2 * sgen-gc.c: Simple generational GC.
5 * Paolo Molaro (lupus@ximian.com)
6 * Rodrigo Kumpera (kumpera@gmail.com)
8 * Copyright 2005-2011 Novell, Inc (http://www.novell.com)
9 * Copyright 2011 Xamarin Inc (http://www.xamarin.com)
11 * Thread start/stop adapted from Boehm's GC:
12 * Copyright (c) 1994 by Xerox Corporation. All rights reserved.
13 * Copyright (c) 1996 by Silicon Graphics. All rights reserved.
14 * Copyright (c) 1998 by Fergus Henderson. All rights reserved.
15 * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved.
16 * Copyright 2001-2003 Ximian, Inc
17 * Copyright 2003-2010 Novell, Inc.
18 * Copyright 2011 Xamarin, Inc.
19 * Copyright (C) 2012 Xamarin Inc
21 * This library is free software; you can redistribute it and/or
22 * modify it under the terms of the GNU Library General Public
23 * License 2.0 as published by the Free Software Foundation;
25 * This library is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
28 * Library General Public License for more details.
30 * You should have received a copy of the GNU Library General Public
31 * License 2.0 along with this library; if not, write to the Free
32 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
34 * Important: allocation provides always zeroed memory, having to do
35 * a memset after allocation is deadly for performance.
36 * Memory usage at startup is currently as follows:
38 * 64 KB internal space
40 * We should provide a small memory config with half the sizes
42 * We currently try to make as few mono assumptions as possible:
43 * 1) 2-word header with no GC pointers in it (first vtable, second to store the
45 * 2) gc descriptor is the second word in the vtable (first word in the class)
46 * 3) 8 byte alignment is the minimum and enough (not true for special structures (SIMD), FIXME)
47 * 4) there is a function to get an object's size and the number of
48 * elements in an array.
49 * 5) we know the special way bounds are allocated for complex arrays
50 * 6) we know about proxies and how to treat them when domains are unloaded
52 * Always try to keep stack usage to a minimum: no recursive behaviour
53 * and no large stack allocs.
55 * General description.
56 * Objects are initially allocated in a nursery using a fast bump-pointer technique.
57 * When the nursery is full we start a nursery collection: this is performed with a
59 * When the old generation is full we start a copying GC of the old generation as well:
60 * this will be changed to mark&sweep with copying when fragmentation becomes to severe
61 * in the future. Maybe we'll even do both during the same collection like IMMIX.
63 * The things that complicate this description are:
64 * *) pinned objects: we can't move them so we need to keep track of them
65 * *) no precise info of the thread stacks and registers: we need to be able to
66 * quickly find the objects that may be referenced conservatively and pin them
67 * (this makes the first issues more important)
68 * *) large objects are too expensive to be dealt with using copying GC: we handle them
69 * with mark/sweep during major collections
70 * *) some objects need to not move even if they are small (interned strings, Type handles):
71 * we use mark/sweep for them, too: they are not allocated in the nursery, but inside
72 * PinnedChunks regions
78 *) we could have a function pointer in MonoClass to implement
79 customized write barriers for value types
81 *) investigate the stuff needed to advance a thread to a GC-safe
82 point (single-stepping, read from unmapped memory etc) and implement it.
83 This would enable us to inline allocations and write barriers, for example,
84 or at least parts of them, like the write barrier checks.
85 We may need this also for handling precise info on stacks, even simple things
86 as having uninitialized data on the stack and having to wait for the prolog
87 to zero it. Not an issue for the last frame that we scan conservatively.
88 We could always not trust the value in the slots anyway.
90 *) modify the jit to save info about references in stack locations:
91 this can be done just for locals as a start, so that at least
92 part of the stack is handled precisely.
94 *) test/fix endianess issues
96 *) Implement a card table as the write barrier instead of remembered
97 sets? Card tables are not easy to implement with our current
98 memory layout. We have several different kinds of major heap
99 objects: Small objects in regular blocks, small objects in pinned
100 chunks and LOS objects. If we just have a pointer we have no way
101 to tell which kind of object it points into, therefore we cannot
102 know where its card table is. The least we have to do to make
103 this happen is to get rid of write barriers for indirect stores.
106 *) Get rid of write barriers for indirect stores. We can do this by
107 telling the GC to wbarrier-register an object once we do an ldloca
108 or ldelema on it, and to unregister it once it's not used anymore
109 (it can only travel downwards on the stack). The problem with
110 unregistering is that it needs to happen eventually no matter
111 what, even if exceptions are thrown, the thread aborts, etc.
112 Rodrigo suggested that we could do only the registering part and
113 let the collector find out (pessimistically) when it's safe to
114 unregister, namely when the stack pointer of the thread that
115 registered the object is higher than it was when the registering
116 happened. This might make for a good first implementation to get
117 some data on performance.
119 *) Some sort of blacklist support? Blacklists is a concept from the
120 Boehm GC: if during a conservative scan we find pointers to an
121 area which we might use as heap, we mark that area as unusable, so
122 pointer retention by random pinning pointers is reduced.
124 *) experiment with max small object size (very small right now - 2kb,
125 because it's tied to the max freelist size)
127 *) add an option to mmap the whole heap in one chunk: it makes for many
128 simplifications in the checks (put the nursery at the top and just use a single
129 check for inclusion/exclusion): the issue this has is that on 32 bit systems it's
130 not flexible (too much of the address space may be used by default or we can't
131 increase the heap as needed) and we'd need a race-free mechanism to return memory
132 back to the system (mprotect(PROT_NONE) will still keep the memory allocated if it
133 was written to, munmap is needed, but the following mmap may not find the same segment
136 *) memzero the major fragments after restarting the world and optionally a smaller
139 *) investigate having fragment zeroing threads
141 *) separate locks for finalization and other minor stuff to reduce
144 *) try a different copying order to improve memory locality
146 *) a thread abort after a store but before the write barrier will
147 prevent the write barrier from executing
149 *) specialized dynamically generated markers/copiers
151 *) Dynamically adjust TLAB size to the number of threads. If we have
152 too many threads that do allocation, we might need smaller TLABs,
153 and we might get better performance with larger TLABs if we only
154 have a handful of threads. We could sum up the space left in all
155 assigned TLABs and if that's more than some percentage of the
156 nursery size, reduce the TLAB size.
158 *) Explore placing unreachable objects on unused nursery memory.
159 Instead of memset'ng a region to zero, place an int[] covering it.
160 A good place to start is add_nursery_frag. The tricky thing here is
161 placing those objects atomically outside of a collection.
163 *) Allocation should use asymmetric Dekker synchronization:
164 http://blogs.oracle.com/dave/resource/Asymmetric-Dekker-Synchronization.txt
165 This should help weak consistency archs.
172 #define _XOPEN_SOURCE
173 #define _DARWIN_C_SOURCE
179 #ifdef HAVE_PTHREAD_H
182 #ifdef HAVE_PTHREAD_NP_H
183 #include <pthread_np.h>
185 #ifdef HAVE_SEMAPHORE_H
186 #include <semaphore.h>
194 #include "metadata/sgen-gc.h"
195 #include "metadata/metadata-internals.h"
196 #include "metadata/class-internals.h"
197 #include "metadata/gc-internal.h"
198 #include "metadata/object-internals.h"
199 #include "metadata/threads.h"
200 #include "metadata/sgen-cardtable.h"
201 #include "metadata/sgen-protocol.h"
202 #include "metadata/sgen-archdep.h"
203 #include "metadata/sgen-bridge.h"
204 #include "metadata/sgen-memory-governor.h"
205 #include "metadata/sgen-hash-table.h"
206 #include "metadata/mono-gc.h"
207 #include "metadata/method-builder.h"
208 #include "metadata/profiler-private.h"
209 #include "metadata/monitor.h"
210 #include "metadata/mempool-internals.h"
211 #include "metadata/marshal.h"
212 #include "metadata/runtime.h"
213 #include "metadata/sgen-cardtable.h"
214 #include "metadata/sgen-pinning.h"
215 #include "metadata/sgen-workers.h"
216 #include "metadata/sgen-layout-stats.h"
217 #include "utils/mono-mmap.h"
218 #include "utils/mono-time.h"
219 #include "utils/mono-semaphore.h"
220 #include "utils/mono-counters.h"
221 #include "utils/mono-proclib.h"
222 #include "utils/mono-memory-model.h"
223 #include "utils/mono-logger-internal.h"
224 #include "utils/dtrace.h"
226 #include <mono/utils/mono-logger-internal.h>
227 #include <mono/utils/memcheck.h>
229 #if defined(__MACH__)
230 #include "utils/mach-support.h"
233 #define OPDEF(a,b,c,d,e,f,g,h,i,j) \
237 #include "mono/cil/opcode.def"
243 #undef pthread_create
245 #undef pthread_detach
248 * ######################################################################
249 * ######## Types and constants used by the GC.
250 * ######################################################################
253 /* 0 means not initialized, 1 is initialized, -1 means in progress */
254 static int gc_initialized = 0;
255 /* If set, check if we need to do something every X allocations */
256 gboolean has_per_allocation_action;
257 /* If set, do a heap check every X allocation */
258 guint32 verify_before_allocs = 0;
259 /* If set, do a minor collection before every X allocation */
260 guint32 collect_before_allocs = 0;
261 /* If set, do a whole heap check before each collection */
262 static gboolean whole_heap_check_before_collection = FALSE;
263 /* If set, do a heap consistency check before each minor collection */
264 static gboolean consistency_check_at_minor_collection = FALSE;
265 /* If set, do a mod union consistency check before each finishing collection pause */
266 static gboolean mod_union_consistency_check = FALSE;
267 /* If set, check whether mark bits are consistent after major collections */
268 static gboolean check_mark_bits_after_major_collection = FALSE;
269 /* If set, check that all nursery objects are pinned/not pinned, depending on context */
270 static gboolean check_nursery_objects_pinned = FALSE;
271 /* If set, do a few checks when the concurrent collector is used */
272 static gboolean do_concurrent_checks = FALSE;
273 /* If set, check that there are no references to the domain left at domain unload */
274 static gboolean xdomain_checks = FALSE;
275 /* If not null, dump the heap after each collection into this file */
276 static FILE *heap_dump_file = NULL;
277 /* If set, mark stacks conservatively, even if precise marking is possible */
278 static gboolean conservative_stack_mark = FALSE;
279 /* If set, do a plausibility check on the scan_starts before and after
281 static gboolean do_scan_starts_check = FALSE;
283 * If the major collector is concurrent and this is FALSE, we will
284 * never initiate a synchronous major collection, unless requested via
287 static gboolean allow_synchronous_major = TRUE;
288 static gboolean disable_minor_collections = FALSE;
289 static gboolean disable_major_collections = FALSE;
290 gboolean do_pin_stats = FALSE;
291 static gboolean do_verify_nursery = FALSE;
292 static gboolean do_dump_nursery_content = FALSE;
294 #ifdef HEAVY_STATISTICS
295 long long stat_objects_alloced_degraded = 0;
296 long long stat_bytes_alloced_degraded = 0;
298 long long stat_copy_object_called_nursery = 0;
299 long long stat_objects_copied_nursery = 0;
300 long long stat_copy_object_called_major = 0;
301 long long stat_objects_copied_major = 0;
303 long long stat_scan_object_called_nursery = 0;
304 long long stat_scan_object_called_major = 0;
306 long long stat_slots_allocated_in_vain;
308 long long stat_nursery_copy_object_failed_from_space = 0;
309 long long stat_nursery_copy_object_failed_forwarded = 0;
310 long long stat_nursery_copy_object_failed_pinned = 0;
311 long long stat_nursery_copy_object_failed_to_space = 0;
313 static int stat_wbarrier_add_to_global_remset = 0;
314 static int stat_wbarrier_set_field = 0;
315 static int stat_wbarrier_set_arrayref = 0;
316 static int stat_wbarrier_arrayref_copy = 0;
317 static int stat_wbarrier_generic_store = 0;
318 static int stat_wbarrier_generic_store_atomic = 0;
319 static int stat_wbarrier_set_root = 0;
320 static int stat_wbarrier_value_copy = 0;
321 static int stat_wbarrier_object_copy = 0;
324 static long long stat_pinned_objects = 0;
326 static long long time_minor_pre_collection_fragment_clear = 0;
327 static long long time_minor_pinning = 0;
328 static long long time_minor_scan_remsets = 0;
329 static long long time_minor_scan_pinned = 0;
330 static long long time_minor_scan_registered_roots = 0;
331 static long long time_minor_scan_thread_data = 0;
332 static long long time_minor_finish_gray_stack = 0;
333 static long long time_minor_fragment_creation = 0;
335 static long long time_major_pre_collection_fragment_clear = 0;
336 static long long time_major_pinning = 0;
337 static long long time_major_scan_pinned = 0;
338 static long long time_major_scan_registered_roots = 0;
339 static long long time_major_scan_thread_data = 0;
340 static long long time_major_scan_alloc_pinned = 0;
341 static long long time_major_scan_finalized = 0;
342 static long long time_major_scan_big_objects = 0;
343 static long long time_major_finish_gray_stack = 0;
344 static long long time_major_free_bigobjs = 0;
345 static long long time_major_los_sweep = 0;
346 static long long time_major_sweep = 0;
347 static long long time_major_fragment_creation = 0;
349 static SGEN_TV_DECLARE (time_major_conc_collection_start);
350 static SGEN_TV_DECLARE (time_major_conc_collection_end);
352 static SGEN_TV_DECLARE (last_minor_collection_start_tv);
353 static SGEN_TV_DECLARE (last_minor_collection_end_tv);
355 int gc_debug_level = 0;
358 static MonoGCFinalizerCallbacks fin_callbacks;
362 mono_gc_flush_info (void)
364 fflush (gc_debug_file);
368 #define TV_DECLARE SGEN_TV_DECLARE
369 #define TV_GETTIME SGEN_TV_GETTIME
370 #define TV_ELAPSED SGEN_TV_ELAPSED
371 #define TV_ELAPSED_MS SGEN_TV_ELAPSED_MS
373 SGEN_TV_DECLARE (sgen_init_timestamp);
375 #define ALIGN_TO(val,align) ((((guint64)val) + ((align) - 1)) & ~((align) - 1))
377 NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
379 #define object_is_forwarded SGEN_OBJECT_IS_FORWARDED
380 #define object_is_pinned SGEN_OBJECT_IS_PINNED
381 #define pin_object SGEN_PIN_OBJECT
382 #define unpin_object SGEN_UNPIN_OBJECT
384 #define ptr_in_nursery sgen_ptr_in_nursery
386 #define LOAD_VTABLE SGEN_LOAD_VTABLE
389 safe_name (void* obj)
391 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
392 return vt->klass->name;
395 #define safe_object_get_size sgen_safe_object_get_size
398 sgen_safe_name (void* obj)
400 return safe_name (obj);
404 * ######################################################################
405 * ######## Global data.
406 * ######################################################################
408 LOCK_DECLARE (gc_mutex);
409 gboolean sgen_try_free_some_memory;
411 #define SCAN_START_SIZE SGEN_SCAN_START_SIZE
413 static mword pagesize = 4096;
414 size_t degraded_mode = 0;
416 static mword bytes_pinned_from_failed_allocation = 0;
418 GCMemSection *nursery_section = NULL;
419 static mword lowest_heap_address = ~(mword)0;
420 static mword highest_heap_address = 0;
422 LOCK_DECLARE (sgen_interruption_mutex);
423 static LOCK_DECLARE (pin_queue_mutex);
425 #define LOCK_PIN_QUEUE mono_mutex_lock (&pin_queue_mutex)
426 #define UNLOCK_PIN_QUEUE mono_mutex_unlock (&pin_queue_mutex)
428 typedef struct _FinalizeReadyEntry FinalizeReadyEntry;
429 struct _FinalizeReadyEntry {
430 FinalizeReadyEntry *next;
434 typedef struct _EphemeronLinkNode EphemeronLinkNode;
436 struct _EphemeronLinkNode {
437 EphemeronLinkNode *next;
446 int current_collection_generation = -1;
447 volatile gboolean concurrent_collection_in_progress = FALSE;
449 /* objects that are ready to be finalized */
450 static FinalizeReadyEntry *fin_ready_list = NULL;
451 static FinalizeReadyEntry *critical_fin_list = NULL;
453 static EphemeronLinkNode *ephemeron_list;
455 /* registered roots: the key to the hash is the root start address */
457 * Different kinds of roots are kept separate to speed up pin_from_roots () for example.
459 SgenHashTable roots_hash [ROOT_TYPE_NUM] = {
460 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
461 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL),
462 SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL)
464 static mword roots_size = 0; /* amount of memory in the root set */
466 #define GC_ROOT_NUM 32
468 int count; /* must be the first field */
469 void *objects [GC_ROOT_NUM];
470 int root_types [GC_ROOT_NUM];
471 uintptr_t extra_info [GC_ROOT_NUM];
475 notify_gc_roots (GCRootReport *report)
479 mono_profiler_gc_roots (report->count, report->objects, report->root_types, report->extra_info);
484 add_profile_gc_root (GCRootReport *report, void *object, int rtype, uintptr_t extra_info)
486 if (report->count == GC_ROOT_NUM)
487 notify_gc_roots (report);
488 report->objects [report->count] = object;
489 report->root_types [report->count] = rtype;
490 report->extra_info [report->count++] = (uintptr_t)((MonoVTable*)LOAD_VTABLE (object))->klass;
493 MonoNativeTlsKey thread_info_key;
495 #ifdef HAVE_KW_THREAD
496 __thread SgenThreadInfo *sgen_thread_info;
497 __thread char *stack_end;
500 /* The size of a TLAB */
501 /* The bigger the value, the less often we have to go to the slow path to allocate a new
502 * one, but the more space is wasted by threads not allocating much memory.
504 * FIXME: Make this self-tuning for each thread.
506 guint32 tlab_size = (1024 * 4);
508 #define MAX_SMALL_OBJ_SIZE SGEN_MAX_SMALL_OBJ_SIZE
510 /* Functions supplied by the runtime to be called by the GC */
511 static MonoGCCallbacks gc_callbacks;
513 #define ALLOC_ALIGN SGEN_ALLOC_ALIGN
514 #define ALLOC_ALIGN_BITS SGEN_ALLOC_ALIGN_BITS
516 #define ALIGN_UP SGEN_ALIGN_UP
518 #define MOVED_OBJECTS_NUM 64
519 static void *moved_objects [MOVED_OBJECTS_NUM];
520 static int moved_objects_idx = 0;
522 /* Vtable of the objects used to fill out nursery fragments before a collection */
523 static MonoVTable *array_fill_vtable;
525 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
526 MonoNativeThreadId main_gc_thread = NULL;
529 /*Object was pinned during the current collection*/
530 static mword objects_pinned;
533 * ######################################################################
534 * ######## Macros and function declarations.
535 * ######################################################################
539 align_pointer (void *ptr)
541 mword p = (mword)ptr;
542 p += sizeof (gpointer) - 1;
543 p &= ~ (sizeof (gpointer) - 1);
547 typedef SgenGrayQueue GrayQueue;
549 /* forward declarations */
550 static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue);
551 static void scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx);
552 static void scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx);
553 static void report_finalizer_roots (void);
554 static void report_registered_roots (void);
556 static void pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue);
557 static void finish_gray_stack (int generation, GrayQueue *queue);
559 void mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise);
562 static void init_stats (void);
564 static int mark_ephemerons_in_range (ScanCopyContext ctx);
565 static void clear_unreachable_ephemerons (ScanCopyContext ctx);
566 static void null_ephemerons_for_domain (MonoDomain *domain);
568 static gboolean major_update_or_finish_concurrent_collection (gboolean force_finish);
570 SgenObjectOperations current_object_ops;
571 SgenMajorCollector major_collector;
572 SgenMinorCollector sgen_minor_collector;
573 static GrayQueue gray_queue;
575 static SgenRemeberedSet remset;
577 /* The gray queue to use from the main collection thread. */
578 #define WORKERS_DISTRIBUTE_GRAY_QUEUE (&gray_queue)
581 * The gray queue a worker job must use. If we're not parallel or
582 * concurrent, we use the main gray queue.
584 static SgenGrayQueue*
585 sgen_workers_get_job_gray_queue (WorkerData *worker_data)
587 return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE;
591 gray_queue_redirect (SgenGrayQueue *queue)
593 gboolean wake = FALSE;
597 GrayQueueSection *section = sgen_gray_object_dequeue_section (queue);
600 sgen_section_gray_queue_enqueue (queue->alloc_prepare_data, section);
605 g_assert (concurrent_collection_in_progress);
606 if (sgen_workers_have_started ()) {
607 sgen_workers_wake_up_all ();
609 if (concurrent_collection_in_progress)
610 g_assert (current_collection_generation == -1);
616 sgen_scan_area_with_callback (char *start, char *end, IterateObjectCallbackFunc callback, void *data, gboolean allow_flags)
618 while (start < end) {
622 if (!*(void**)start) {
623 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
628 if (!(obj = SGEN_OBJECT_IS_FORWARDED (start)))
634 size = ALIGN_UP (safe_object_get_size ((MonoObject*)obj));
636 if ((MonoVTable*)SGEN_LOAD_VTABLE (obj) != array_fill_vtable)
637 callback (obj, size, data);
644 need_remove_object_for_domain (char *start, MonoDomain *domain)
646 if (mono_object_domain (start) == domain) {
647 SGEN_LOG (4, "Need to cleanup object %p", start);
648 binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start));
655 process_object_for_domain_clearing (char *start, MonoDomain *domain)
657 GCVTable *vt = (GCVTable*)LOAD_VTABLE (start);
658 if (vt->klass == mono_defaults.internal_thread_class)
659 g_assert (mono_object_domain (start) == mono_get_root_domain ());
660 /* The object could be a proxy for an object in the domain
662 #ifndef DISABLE_REMOTING
663 if (mono_defaults.real_proxy_class->supertypes && mono_class_has_parent_fast (vt->klass, mono_defaults.real_proxy_class)) {
664 MonoObject *server = ((MonoRealProxy*)start)->unwrapped_server;
666 /* The server could already have been zeroed out, so
667 we need to check for that, too. */
668 if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) {
669 SGEN_LOG (4, "Cleaning up remote pointer in %p to object %p", start, server);
670 ((MonoRealProxy*)start)->unwrapped_server = NULL;
677 clear_domain_process_object (char *obj, MonoDomain *domain)
681 process_object_for_domain_clearing (obj, domain);
682 remove = need_remove_object_for_domain (obj, domain);
684 if (remove && ((MonoObject*)obj)->synchronisation) {
685 void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj);
687 sgen_register_disappearing_link (NULL, dislink, FALSE, TRUE);
694 clear_domain_process_minor_object_callback (char *obj, size_t size, MonoDomain *domain)
696 if (clear_domain_process_object (obj, domain))
697 memset (obj, 0, size);
701 clear_domain_process_major_object_callback (char *obj, size_t size, MonoDomain *domain)
703 clear_domain_process_object (obj, domain);
707 clear_domain_free_major_non_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
709 if (need_remove_object_for_domain (obj, domain))
710 major_collector.free_non_pinned_object (obj, size);
714 clear_domain_free_major_pinned_object_callback (char *obj, size_t size, MonoDomain *domain)
716 if (need_remove_object_for_domain (obj, domain))
717 major_collector.free_pinned_object (obj, size);
721 * When appdomains are unloaded we can easily remove objects that have finalizers,
722 * but all the others could still be present in random places on the heap.
723 * We need a sweep to get rid of them even though it's going to be costly
725 * The reason we need to remove them is because we access the vtable and class
726 * structures to know the object size and the reference bitmap: once the domain is
727 * unloaded the point to random memory.
730 mono_gc_clear_domain (MonoDomain * domain)
732 LOSObject *bigobj, *prev;
737 binary_protocol_domain_unload_begin (domain);
741 if (concurrent_collection_in_progress)
742 sgen_perform_collection (0, GENERATION_OLD, "clear domain", TRUE);
743 g_assert (!concurrent_collection_in_progress);
745 sgen_process_fin_stage_entries ();
746 sgen_process_dislink_stage_entries ();
748 sgen_clear_nursery_fragments ();
750 if (xdomain_checks && domain != mono_get_root_domain ()) {
751 sgen_scan_for_registered_roots_in_domain (domain, ROOT_TYPE_NORMAL);
752 sgen_scan_for_registered_roots_in_domain (domain, ROOT_TYPE_WBARRIER);
753 sgen_check_for_xdomain_refs ();
756 /*Ephemerons and dislinks must be processed before LOS since they might end up pointing
757 to memory returned to the OS.*/
758 null_ephemerons_for_domain (domain);
760 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
761 sgen_null_links_for_domain (domain, i);
763 for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i)
764 sgen_remove_finalizers_for_domain (domain, i);
766 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
767 (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain, FALSE);
769 /* We need two passes over major and large objects because
770 freeing such objects might give their memory back to the OS
771 (in the case of large objects) or obliterate its vtable
772 (pinned objects with major-copying or pinned and non-pinned
773 objects with major-mark&sweep), but we might need to
774 dereference a pointer from an object to another object if
775 the first object is a proxy. */
776 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, (IterateObjectCallbackFunc)clear_domain_process_major_object_callback, domain);
777 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
778 clear_domain_process_object (bigobj->data, domain);
781 for (bigobj = los_object_list; bigobj;) {
782 if (need_remove_object_for_domain (bigobj->data, domain)) {
783 LOSObject *to_free = bigobj;
785 prev->next = bigobj->next;
787 los_object_list = bigobj->next;
788 bigobj = bigobj->next;
789 SGEN_LOG (4, "Freeing large object %p", bigobj->data);
790 sgen_los_free_object (to_free);
794 bigobj = bigobj->next;
796 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_NON_PINNED, (IterateObjectCallbackFunc)clear_domain_free_major_non_pinned_object_callback, domain);
797 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_PINNED, (IterateObjectCallbackFunc)clear_domain_free_major_pinned_object_callback, domain);
799 if (domain == mono_get_root_domain ()) {
800 if (G_UNLIKELY (do_pin_stats))
801 sgen_pin_stats_print_class_stats ();
802 sgen_object_layout_dump (stdout);
805 sgen_restart_world (0, NULL);
807 binary_protocol_domain_unload_end (domain);
813 * sgen_add_to_global_remset:
815 * The global remset contains locations which point into newspace after
816 * a minor collection. This can happen if the objects they point to are pinned.
818 * LOCKING: If called from a parallel collector, the global remset
819 * lock must be held. For serial collectors that is not necessary.
822 sgen_add_to_global_remset (gpointer ptr, gpointer obj)
824 SGEN_ASSERT (5, sgen_ptr_in_nursery (obj), "Target pointer of global remset must be in the nursery");
826 HEAVY_STAT (++stat_wbarrier_add_to_global_remset);
828 if (!major_collector.is_concurrent) {
829 SGEN_ASSERT (5, current_collection_generation != -1, "Global remsets can only be added during collections");
831 if (current_collection_generation == -1)
832 SGEN_ASSERT (5, sgen_concurrent_collection_in_progress (), "Global remsets outside of collection pauses can only be added by the concurrent collector");
835 if (!object_is_pinned (obj))
836 SGEN_ASSERT (5, sgen_minor_collector.is_split || sgen_concurrent_collection_in_progress (), "Non-pinned objects can only remain in nursery if it is a split nursery");
837 else if (sgen_cement_lookup_or_register (obj))
840 remset.record_pointer (ptr);
842 if (G_UNLIKELY (do_pin_stats))
843 sgen_pin_stats_register_global_remset (obj);
845 SGEN_LOG (8, "Adding global remset for %p", ptr);
846 binary_protocol_global_remset (ptr, obj, (gpointer)SGEN_LOAD_VTABLE (obj));
850 if (G_UNLIKELY (MONO_GC_GLOBAL_REMSET_ADD_ENABLED ())) {
851 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
852 MONO_GC_GLOBAL_REMSET_ADD ((mword)ptr, (mword)obj, sgen_safe_object_get_size (obj),
853 vt->klass->name_space, vt->klass->name);
859 * sgen_drain_gray_stack:
861 * Scan objects in the gray stack until the stack is empty. This should be called
862 * frequently after each object is copied, to achieve better locality and cache
865 * max_objs is the maximum number of objects to scan, or -1 to scan until the stack is
869 sgen_drain_gray_stack (int max_objs, ScanCopyContext ctx)
871 ScanObjectFunc scan_func = ctx.scan_func;
872 GrayQueue *queue = ctx.queue;
876 for (i = 0; i != max_objs; ++i) {
879 GRAY_OBJECT_DEQUEUE (queue, &obj, &desc);
882 SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj));
883 scan_func (obj, desc, queue);
885 } while (max_objs < 0);
890 * Addresses in the pin queue are already sorted. This function finds
891 * the object header for each address and pins the object. The
892 * addresses must be inside the nursery section. The (start of the)
893 * address array is overwritten with the addresses of the actually
894 * pinned objects. Return the number of pinned objects.
897 pin_objects_from_nursery_pin_queue (ScanCopyContext ctx)
899 GCMemSection *section = nursery_section;
900 void **start = section->pin_queue_start;
901 void **end = start + section->pin_queue_num_entries;
902 void *start_nursery = section->data;
903 void *end_nursery = section->next_data;
908 void *pinning_front = start_nursery;
910 void **definitely_pinned = start;
911 ScanObjectFunc scan_func = ctx.scan_func;
912 SgenGrayQueue *queue = ctx.queue;
914 sgen_nursery_allocator_prepare_for_pinning ();
916 while (start < end) {
917 void *obj_to_pin = NULL;
918 size_t obj_to_pin_size = 0;
923 SGEN_ASSERT (0, addr >= start_nursery && addr < end_nursery, "Potential pinning address out of range");
924 SGEN_ASSERT (0, addr >= last, "Pin queue not sorted");
931 SGEN_LOG (5, "Considering pinning addr %p", addr);
932 /* We've already processed everything up to pinning_front. */
933 if (addr < pinning_front) {
939 * Find the closest scan start <= addr. We might search backward in the
940 * scan_starts array because entries might be NULL. In the worst case we
941 * start at start_nursery.
943 idx = ((char*)addr - (char*)section->data) / SCAN_START_SIZE;
944 SGEN_ASSERT (0, idx < section->num_scan_start, "Scan start index out of range");
945 search_start = (void*)section->scan_starts [idx];
946 if (!search_start || search_start > addr) {
949 search_start = section->scan_starts [idx];
950 if (search_start && search_start <= addr)
953 if (!search_start || search_start > addr)
954 search_start = start_nursery;
958 * If the pinning front is closer than the scan start we found, start
959 * searching at the front.
961 if (search_start < pinning_front)
962 search_start = pinning_front;
965 * Now addr should be in an object a short distance from search_start.
967 * search_start must point to zeroed mem or point to an object.
973 if (!*(void**)search_start) {
974 search_start = (void*)ALIGN_UP ((mword)search_start + sizeof (gpointer));
975 /* The loop condition makes sure we don't overrun addr. */
979 obj_size = ALIGN_UP (safe_object_get_size ((MonoObject*)search_start));
981 if (addr >= search_start && (char*)addr < (char*)search_start + obj_size) {
982 /* This is the object we're looking for. */
983 obj_to_pin = search_start;
984 obj_to_pin_size = obj_size;
988 /* Skip to the next object */
989 search_start = (void*)((char*)search_start + obj_size);
990 } while (search_start <= addr);
992 /* We've searched past the address we were looking for. */
994 pinning_front = search_start;
995 goto next_pin_queue_entry;
999 * We've found an object to pin. It might still be a dummy array, but we
1000 * can advance the pinning front in any case.
1002 pinning_front = (char*)obj_to_pin + obj_to_pin_size;
1005 * If this is a dummy array marking the beginning of a nursery
1006 * fragment, we don't pin it.
1008 if (((MonoObject*)obj_to_pin)->synchronisation == GINT_TO_POINTER (-1))
1009 goto next_pin_queue_entry;
1012 * Finally - pin the object!
1014 desc = sgen_obj_get_descriptor_safe (obj_to_pin);
1016 scan_func (obj_to_pin, desc, queue);
1018 SGEN_LOG (4, "Pinned object %p, vtable %p (%s), count %d\n",
1019 obj_to_pin, *(void**)obj_to_pin, safe_name (obj_to_pin), count);
1020 binary_protocol_pin (obj_to_pin,
1021 (gpointer)LOAD_VTABLE (obj_to_pin),
1022 safe_object_get_size (obj_to_pin));
1024 #ifdef ENABLE_DTRACE
1025 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1026 int gen = sgen_ptr_in_nursery (obj_to_pin) ? GENERATION_NURSERY : GENERATION_OLD;
1027 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj_to_pin);
1028 MONO_GC_OBJ_PINNED ((mword)obj_to_pin,
1029 sgen_safe_object_get_size (obj_to_pin),
1030 vt->klass->name_space, vt->klass->name, gen);
1034 pin_object (obj_to_pin);
1035 GRAY_OBJECT_ENQUEUE (queue, obj_to_pin, desc);
1036 if (G_UNLIKELY (do_pin_stats))
1037 sgen_pin_stats_register_object (obj_to_pin, obj_to_pin_size);
1038 definitely_pinned [count] = obj_to_pin;
1042 next_pin_queue_entry:
1046 //printf ("effective pinned: %d (at the end: %d)\n", count, (char*)end_nursery - (char*)last);
1047 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS) {
1048 GCRootReport report;
1050 for (idx = 0; idx < count; ++idx)
1051 add_profile_gc_root (&report, definitely_pinned [idx], MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
1052 notify_gc_roots (&report);
1054 stat_pinned_objects += count;
1059 pin_objects_in_nursery (ScanCopyContext ctx)
1063 if (!nursery_section->pin_queue_num_entries)
1066 reduced_to = pin_objects_from_nursery_pin_queue (ctx);
1067 nursery_section->pin_queue_num_entries = reduced_to;
1069 nursery_section->pin_queue_start = NULL;
1074 sgen_pin_object (void *object, GrayQueue *queue)
1076 g_assert (!concurrent_collection_in_progress);
1078 SGEN_PIN_OBJECT (object);
1079 sgen_pin_stage_ptr (object);
1081 if (G_UNLIKELY (do_pin_stats))
1082 sgen_pin_stats_register_object (object, safe_object_get_size (object));
1084 GRAY_OBJECT_ENQUEUE (queue, object, sgen_obj_get_descriptor (object));
1085 binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object));
1087 #ifdef ENABLE_DTRACE
1088 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
1089 int gen = sgen_ptr_in_nursery (object) ? GENERATION_NURSERY : GENERATION_OLD;
1090 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (object);
1091 MONO_GC_OBJ_PINNED ((mword)object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen);
1097 sgen_parallel_pin_or_update (void **ptr, void *obj, MonoVTable *vt, SgenGrayQueue *queue)
1101 gboolean major_pinned = FALSE;
1103 if (sgen_ptr_in_nursery (obj)) {
1104 if (SGEN_CAS_PTR (obj, (void*)((mword)vt | SGEN_PINNED_BIT), vt) == vt) {
1105 sgen_pin_object (obj, queue);
1109 major_collector.pin_major_object (obj, queue);
1110 major_pinned = TRUE;
1113 vtable_word = *(mword*)obj;
1114 /*someone else forwarded it, update the pointer and bail out*/
1115 if (vtable_word & SGEN_FORWARDED_BIT) {
1116 *ptr = (void*)(vtable_word & ~SGEN_VTABLE_BITS_MASK);
1120 /*someone pinned it, nothing to do.*/
1121 if (vtable_word & SGEN_PINNED_BIT || major_pinned)
1126 /* Sort the addresses in array in increasing order.
1127 * Done using a by-the book heap sort. Which has decent and stable performance, is pretty cache efficient.
1130 sgen_sort_addresses (void **array, size_t size)
1135 for (i = 1; i < size; ++i) {
1138 size_t parent = (child - 1) / 2;
1140 if (array [parent] >= array [child])
1143 tmp = array [parent];
1144 array [parent] = array [child];
1145 array [child] = tmp;
1151 for (i = size - 1; i > 0; --i) {
1154 array [i] = array [0];
1160 while (root * 2 + 1 <= end) {
1161 size_t child = root * 2 + 1;
1163 if (child < end && array [child] < array [child + 1])
1165 if (array [root] >= array [child])
1169 array [root] = array [child];
1170 array [child] = tmp;
1178 * Scan the memory between start and end and queue values which could be pointers
1179 * to the area between start_nursery and end_nursery for later consideration.
1180 * Typically used for thread stacks.
1183 conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type)
1187 #ifdef VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE
1188 VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE (start, (char*)end - (char*)start);
1191 while (start < end) {
1192 if (*start >= start_nursery && *start < end_nursery) {
1194 * *start can point to the middle of an object
1195 * note: should we handle pointing at the end of an object?
1196 * pinning in C# code disallows pointing at the end of an object
1197 * but there is some small chance that an optimizing C compiler
1198 * may keep the only reference to an object by pointing
1199 * at the end of it. We ignore this small chance for now.
1200 * Pointers to the end of an object are indistinguishable
1201 * from pointers to the start of the next object in memory
1202 * so if we allow that we'd need to pin two objects...
1203 * We queue the pointer in an array, the
1204 * array will then be sorted and uniqued. This way
1205 * we can coalesce several pinning pointers and it should
1206 * be faster since we'd do a memory scan with increasing
1207 * addresses. Note: we can align the address to the allocation
1208 * alignment, so the unique process is more effective.
1210 mword addr = (mword)*start;
1211 addr &= ~(ALLOC_ALIGN - 1);
1212 if (addr >= (mword)start_nursery && addr < (mword)end_nursery) {
1213 SGEN_LOG (6, "Pinning address %p from %p", (void*)addr, start);
1214 sgen_pin_stage_ptr ((void*)addr);
1217 if (G_UNLIKELY (do_pin_stats)) {
1218 if (ptr_in_nursery ((void*)addr))
1219 sgen_pin_stats_register_address ((char*)addr, pin_type);
1225 SGEN_LOG (7, "found %d potential pinned heap pointers", count);
1229 * The first thing we do in a collection is to identify pinned objects.
1230 * This function considers all the areas of memory that need to be
1231 * conservatively scanned.
1234 pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue)
1238 SGEN_LOG (2, "Scanning pinned roots (%d bytes, %d/%d entries)", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries);
1239 /* objects pinned from the API are inside these roots */
1240 SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], start_root, root) {
1241 SGEN_LOG (6, "Pinned roots %p-%p", start_root, root->end_root);
1242 conservatively_pin_objects_from (start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER);
1243 } SGEN_HASH_TABLE_FOREACH_END;
1244 /* now deal with the thread stacks
1245 * in the future we should be able to conservatively scan only:
1246 * *) the cpu registers
1247 * *) the unmanaged stack frames
1248 * *) the _last_ managed stack frame
1249 * *) pointers slots in managed frames
1251 scan_thread_data (start_nursery, end_nursery, FALSE, queue);
1255 unpin_objects_from_queue (SgenGrayQueue *queue)
1260 GRAY_OBJECT_DEQUEUE (queue, &addr, &desc);
1263 g_assert (SGEN_OBJECT_IS_PINNED (addr));
1264 SGEN_UNPIN_OBJECT (addr);
1269 CopyOrMarkObjectFunc func;
1271 } UserCopyOrMarkData;
1274 single_arg_user_copy_or_mark (void **obj, void *gc_data)
1276 UserCopyOrMarkData *data = gc_data;
1278 data->func (obj, data->queue);
1282 * The memory area from start_root to end_root contains pointers to objects.
1283 * Their position is precisely described by @desc (this means that the pointer
1284 * can be either NULL or the pointer to the start of an object).
1285 * This functions copies them to to_space updates them.
1287 * This function is not thread-safe!
1290 precisely_scan_objects_from (void** start_root, void** end_root, char* n_start, char *n_end, mword desc, ScanCopyContext ctx)
1292 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1293 SgenGrayQueue *queue = ctx.queue;
1295 switch (desc & ROOT_DESC_TYPE_MASK) {
1296 case ROOT_DESC_BITMAP:
1297 desc >>= ROOT_DESC_TYPE_SHIFT;
1299 if ((desc & 1) && *start_root) {
1300 copy_func (start_root, queue);
1301 SGEN_LOG (9, "Overwrote root at %p with %p", start_root, *start_root);
1302 sgen_drain_gray_stack (-1, ctx);
1308 case ROOT_DESC_COMPLEX: {
1309 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1310 gsize bwords = (*bitmap_data) - 1;
1311 void **start_run = start_root;
1313 while (bwords-- > 0) {
1314 gsize bmap = *bitmap_data++;
1315 void **objptr = start_run;
1317 if ((bmap & 1) && *objptr) {
1318 copy_func (objptr, queue);
1319 SGEN_LOG (9, "Overwrote root at %p with %p", objptr, *objptr);
1320 sgen_drain_gray_stack (-1, ctx);
1325 start_run += GC_BITS_PER_WORD;
1329 case ROOT_DESC_USER: {
1330 UserCopyOrMarkData data = { copy_func, queue };
1331 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1332 marker (start_root, single_arg_user_copy_or_mark, &data);
1335 case ROOT_DESC_RUN_LEN:
1336 g_assert_not_reached ();
1338 g_assert_not_reached ();
1343 reset_heap_boundaries (void)
1345 lowest_heap_address = ~(mword)0;
1346 highest_heap_address = 0;
1350 sgen_update_heap_boundaries (mword low, mword high)
1355 old = lowest_heap_address;
1358 } while (SGEN_CAS_PTR ((gpointer*)&lowest_heap_address, (gpointer)low, (gpointer)old) != (gpointer)old);
1361 old = highest_heap_address;
1364 } while (SGEN_CAS_PTR ((gpointer*)&highest_heap_address, (gpointer)high, (gpointer)old) != (gpointer)old);
1368 * Allocate and setup the data structures needed to be able to allocate objects
1369 * in the nursery. The nursery is stored in nursery_section.
1372 alloc_nursery (void)
1374 GCMemSection *section;
1379 if (nursery_section)
1381 SGEN_LOG (2, "Allocating nursery size: %zu", (size_t)sgen_nursery_size);
1382 /* later we will alloc a larger area for the nursery but only activate
1383 * what we need. The rest will be used as expansion if we have too many pinned
1384 * objects in the existing nursery.
1386 /* FIXME: handle OOM */
1387 section = sgen_alloc_internal (INTERNAL_MEM_SECTION);
1389 alloc_size = sgen_nursery_size;
1391 /* If there isn't enough space even for the nursery we should simply abort. */
1392 g_assert (sgen_memgov_try_alloc_space (alloc_size, SPACE_NURSERY));
1394 #ifdef SGEN_ALIGN_NURSERY
1395 data = major_collector.alloc_heap (alloc_size, alloc_size, DEFAULT_NURSERY_BITS);
1397 data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS);
1399 sgen_update_heap_boundaries ((mword)data, (mword)(data + sgen_nursery_size));
1400 SGEN_LOG (4, "Expanding nursery size (%p-%p): %lu, total: %lu", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ());
1401 section->data = section->next_data = data;
1402 section->size = alloc_size;
1403 section->end_data = data + sgen_nursery_size;
1404 scan_starts = (alloc_size + SCAN_START_SIZE - 1) / SCAN_START_SIZE;
1405 section->scan_starts = sgen_alloc_internal_dynamic (sizeof (char*) * scan_starts, INTERNAL_MEM_SCAN_STARTS, TRUE);
1406 section->num_scan_start = scan_starts;
1408 nursery_section = section;
1410 sgen_nursery_allocator_set_nursery_bounds (data, data + sgen_nursery_size);
1414 mono_gc_get_nursery (int *shift_bits, size_t *size)
1416 *size = sgen_nursery_size;
1417 #ifdef SGEN_ALIGN_NURSERY
1418 *shift_bits = DEFAULT_NURSERY_BITS;
1422 return sgen_get_nursery_start ();
1426 mono_gc_set_current_thread_appdomain (MonoDomain *domain)
1428 SgenThreadInfo *info = mono_thread_info_current ();
1430 /* Could be called from sgen_thread_unregister () with a NULL info */
1433 info->stopped_domain = domain;
1438 mono_gc_precise_stack_mark_enabled (void)
1440 return !conservative_stack_mark;
1444 mono_gc_get_logfile (void)
1446 return gc_debug_file;
1450 report_finalizer_roots_list (FinalizeReadyEntry *list)
1452 GCRootReport report;
1453 FinalizeReadyEntry *fin;
1456 for (fin = list; fin; fin = fin->next) {
1459 add_profile_gc_root (&report, fin->object, MONO_PROFILE_GC_ROOT_FINALIZER, 0);
1461 notify_gc_roots (&report);
1465 report_finalizer_roots (void)
1467 report_finalizer_roots_list (fin_ready_list);
1468 report_finalizer_roots_list (critical_fin_list);
1471 static GCRootReport *root_report;
1474 single_arg_report_root (void **obj, void *gc_data)
1477 add_profile_gc_root (root_report, *obj, MONO_PROFILE_GC_ROOT_OTHER, 0);
1481 precisely_report_roots_from (GCRootReport *report, void** start_root, void** end_root, mword desc)
1483 switch (desc & ROOT_DESC_TYPE_MASK) {
1484 case ROOT_DESC_BITMAP:
1485 desc >>= ROOT_DESC_TYPE_SHIFT;
1487 if ((desc & 1) && *start_root) {
1488 add_profile_gc_root (report, *start_root, MONO_PROFILE_GC_ROOT_OTHER, 0);
1494 case ROOT_DESC_COMPLEX: {
1495 gsize *bitmap_data = sgen_get_complex_descriptor_bitmap (desc);
1496 gsize bwords = (*bitmap_data) - 1;
1497 void **start_run = start_root;
1499 while (bwords-- > 0) {
1500 gsize bmap = *bitmap_data++;
1501 void **objptr = start_run;
1503 if ((bmap & 1) && *objptr) {
1504 add_profile_gc_root (report, *objptr, MONO_PROFILE_GC_ROOT_OTHER, 0);
1509 start_run += GC_BITS_PER_WORD;
1513 case ROOT_DESC_USER: {
1514 MonoGCRootMarkFunc marker = sgen_get_user_descriptor_func (desc);
1515 root_report = report;
1516 marker (start_root, single_arg_report_root, NULL);
1519 case ROOT_DESC_RUN_LEN:
1520 g_assert_not_reached ();
1522 g_assert_not_reached ();
1527 report_registered_roots_by_type (int root_type)
1529 GCRootReport report;
1533 SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1534 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1535 precisely_report_roots_from (&report, start_root, (void**)root->end_root, root->root_desc);
1536 } SGEN_HASH_TABLE_FOREACH_END;
1537 notify_gc_roots (&report);
1541 report_registered_roots (void)
1543 report_registered_roots_by_type (ROOT_TYPE_NORMAL);
1544 report_registered_roots_by_type (ROOT_TYPE_WBARRIER);
1548 scan_finalizer_entries (FinalizeReadyEntry *list, ScanCopyContext ctx)
1550 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
1551 SgenGrayQueue *queue = ctx.queue;
1552 FinalizeReadyEntry *fin;
1554 for (fin = list; fin; fin = fin->next) {
1557 SGEN_LOG (5, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object));
1558 copy_func (&fin->object, queue);
1563 generation_name (int generation)
1565 switch (generation) {
1566 case GENERATION_NURSERY: return "nursery";
1567 case GENERATION_OLD: return "old";
1568 default: g_assert_not_reached ();
1573 sgen_generation_name (int generation)
1575 return generation_name (generation);
1578 SgenObjectOperations *
1579 sgen_get_current_object_ops (void){
1580 return ¤t_object_ops;
1585 finish_gray_stack (int generation, GrayQueue *queue)
1589 int done_with_ephemerons, ephemeron_rounds = 0;
1590 CopyOrMarkObjectFunc copy_func = current_object_ops.copy_or_mark_object;
1591 ScanObjectFunc scan_func = current_object_ops.scan_object;
1592 ScanCopyContext ctx = { scan_func, copy_func, queue };
1593 char *start_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_start () : NULL;
1594 char *end_addr = generation == GENERATION_NURSERY ? sgen_get_nursery_end () : (char*)-1;
1597 * We copied all the reachable objects. Now it's the time to copy
1598 * the objects that were not referenced by the roots, but by the copied objects.
1599 * we built a stack of objects pointed to by gray_start: they are
1600 * additional roots and we may add more items as we go.
1601 * We loop until gray_start == gray_objects which means no more objects have
1602 * been added. Note this is iterative: no recursion is involved.
1603 * We need to walk the LO list as well in search of marked big objects
1604 * (use a flag since this is needed only on major collections). We need to loop
1605 * here as well, so keep a counter of marked LO (increasing it in copy_object).
1606 * To achieve better cache locality and cache usage, we drain the gray stack
1607 * frequently, after each object is copied, and just finish the work here.
1609 sgen_drain_gray_stack (-1, ctx);
1611 SGEN_LOG (2, "%s generation done", generation_name (generation));
1614 Reset bridge data, we might have lingering data from a previous collection if this is a major
1615 collection trigged by minor overflow.
1617 We must reset the gathered bridges since their original block might be evacuated due to major
1618 fragmentation in the meanwhile and the bridge code should not have to deal with that.
1620 if (sgen_need_bridge_processing ())
1621 sgen_bridge_reset_data ();
1624 * Walk the ephemeron tables marking all values with reachable keys. This must be completely done
1625 * before processing finalizable objects and non-tracking weak links to avoid finalizing/clearing
1626 * objects that are in fact reachable.
1628 done_with_ephemerons = 0;
1630 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1631 sgen_drain_gray_stack (-1, ctx);
1633 } while (!done_with_ephemerons);
1635 sgen_mark_togglerefs (start_addr, end_addr, ctx);
1637 if (sgen_need_bridge_processing ()) {
1638 /*Make sure the gray stack is empty before we process bridge objects so we get liveness right*/
1639 sgen_drain_gray_stack (-1, ctx);
1640 sgen_collect_bridge_objects (generation, ctx);
1641 if (generation == GENERATION_OLD)
1642 sgen_collect_bridge_objects (GENERATION_NURSERY, ctx);
1645 Do the first bridge step here, as the collector liveness state will become useless after that.
1647 An important optimization is to only proccess the possibly dead part of the object graph and skip
1648 over all live objects as we transitively know everything they point must be alive too.
1650 The above invariant is completely wrong if we let the gray queue be drained and mark/copy everything.
1652 This has the unfortunate side effect of making overflow collections perform the first step twice, but
1653 given we now have heuristics that perform major GC in anticipation of minor overflows this should not
1656 sgen_bridge_processing_stw_step ();
1660 Make sure we drain the gray stack before processing disappearing links and finalizers.
1661 If we don't make sure it is empty we might wrongly see a live object as dead.
1663 sgen_drain_gray_stack (-1, ctx);
1666 We must clear weak links that don't track resurrection before processing object ready for
1667 finalization so they can be cleared before that.
1669 sgen_null_link_in_range (generation, TRUE, ctx);
1670 if (generation == GENERATION_OLD)
1671 sgen_null_link_in_range (GENERATION_NURSERY, TRUE, ctx);
1674 /* walk the finalization queue and move also the objects that need to be
1675 * finalized: use the finalized objects as new roots so the objects they depend
1676 * on are also not reclaimed. As with the roots above, only objects in the nursery
1677 * are marked/copied.
1679 sgen_finalize_in_range (generation, ctx);
1680 if (generation == GENERATION_OLD)
1681 sgen_finalize_in_range (GENERATION_NURSERY, ctx);
1682 /* drain the new stack that might have been created */
1683 SGEN_LOG (6, "Precise scan of gray area post fin");
1684 sgen_drain_gray_stack (-1, ctx);
1687 * This must be done again after processing finalizable objects since CWL slots are cleared only after the key is finalized.
1689 done_with_ephemerons = 0;
1691 done_with_ephemerons = mark_ephemerons_in_range (ctx);
1692 sgen_drain_gray_stack (-1, ctx);
1694 } while (!done_with_ephemerons);
1697 * Clear ephemeron pairs with unreachable keys.
1698 * We pass the copy func so we can figure out if an array was promoted or not.
1700 clear_unreachable_ephemerons (ctx);
1703 * We clear togglerefs only after all possible chances of revival are done.
1704 * This is semantically more inline with what users expect and it allows for
1705 * user finalizers to correctly interact with TR objects.
1707 sgen_clear_togglerefs (start_addr, end_addr, ctx);
1710 SGEN_LOG (2, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds);
1713 * handle disappearing links
1714 * Note we do this after checking the finalization queue because if an object
1715 * survives (at least long enough to be finalized) we don't clear the link.
1716 * This also deals with a possible issue with the monitor reclamation: with the Boehm
1717 * GC a finalized object my lose the monitor because it is cleared before the finalizer is
1720 g_assert (sgen_gray_object_queue_is_empty (queue));
1722 sgen_null_link_in_range (generation, FALSE, ctx);
1723 if (generation == GENERATION_OLD)
1724 sgen_null_link_in_range (GENERATION_NURSERY, FALSE, ctx);
1725 if (sgen_gray_object_queue_is_empty (queue))
1727 sgen_drain_gray_stack (-1, ctx);
1730 g_assert (sgen_gray_object_queue_is_empty (queue));
1732 sgen_gray_object_queue_trim_free_list (queue);
1736 sgen_check_section_scan_starts (GCMemSection *section)
1739 for (i = 0; i < section->num_scan_start; ++i) {
1740 if (section->scan_starts [i]) {
1741 mword size = safe_object_get_size ((MonoObject*) section->scan_starts [i]);
1742 g_assert (size >= sizeof (MonoObject) && size <= MAX_SMALL_OBJ_SIZE);
1748 check_scan_starts (void)
1750 if (!do_scan_starts_check)
1752 sgen_check_section_scan_starts (nursery_section);
1753 major_collector.check_scan_starts ();
1757 scan_from_registered_roots (char *addr_start, char *addr_end, int root_type, ScanCopyContext ctx)
1761 SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) {
1762 SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc);
1763 precisely_scan_objects_from (start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, ctx);
1764 } SGEN_HASH_TABLE_FOREACH_END;
1768 sgen_dump_occupied (char *start, char *end, char *section_start)
1770 fprintf (heap_dump_file, "<occupied offset=\"%td\" size=\"%td\"/>\n", start - section_start, end - start);
1774 sgen_dump_section (GCMemSection *section, const char *type)
1776 char *start = section->data;
1777 char *end = section->data + section->size;
1778 char *occ_start = NULL;
1780 char *old_start = NULL; /* just for debugging */
1782 fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
1784 while (start < end) {
1788 if (!*(void**)start) {
1790 sgen_dump_occupied (occ_start, start, section->data);
1793 start += sizeof (void*); /* should be ALLOC_ALIGN, really */
1796 g_assert (start < section->next_data);
1801 vt = (GCVTable*)LOAD_VTABLE (start);
1804 size = ALIGN_UP (safe_object_get_size ((MonoObject*) start));
1807 fprintf (heap_dump_file, "<object offset=\"%d\" class=\"%s.%s\" size=\"%d\"/>\n",
1808 start - section->data,
1809 vt->klass->name_space, vt->klass->name,
1817 sgen_dump_occupied (occ_start, start, section->data);
1819 fprintf (heap_dump_file, "</section>\n");
1823 dump_object (MonoObject *obj, gboolean dump_location)
1825 static char class_name [1024];
1827 MonoClass *class = mono_object_class (obj);
1831 * Python's XML parser is too stupid to parse angle brackets
1832 * in strings, so we just ignore them;
1835 while (class->name [i] && j < sizeof (class_name) - 1) {
1836 if (!strchr ("<>\"", class->name [i]))
1837 class_name [j++] = class->name [i];
1840 g_assert (j < sizeof (class_name));
1843 fprintf (heap_dump_file, "<object class=\"%s.%s\" size=\"%zd\"",
1844 class->name_space, class_name,
1845 safe_object_get_size (obj));
1846 if (dump_location) {
1847 const char *location;
1848 if (ptr_in_nursery (obj))
1849 location = "nursery";
1850 else if (safe_object_get_size (obj) <= MAX_SMALL_OBJ_SIZE)
1854 fprintf (heap_dump_file, " location=\"%s\"", location);
1856 fprintf (heap_dump_file, "/>\n");
1860 dump_heap (const char *type, int num, const char *reason)
1865 fprintf (heap_dump_file, "<collection type=\"%s\" num=\"%d\"", type, num);
1867 fprintf (heap_dump_file, " reason=\"%s\"", reason);
1868 fprintf (heap_dump_file, ">\n");
1869 fprintf (heap_dump_file, "<other-mem-usage type=\"mempools\" size=\"%ld\"/>\n", mono_mempool_get_bytes_allocated ());
1870 sgen_dump_internal_mem_usage (heap_dump_file);
1871 fprintf (heap_dump_file, "<pinned type=\"stack\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_STACK));
1872 /* fprintf (heap_dump_file, "<pinned type=\"static-data\" bytes=\"%d\"/>\n", pinned_byte_counts [PIN_TYPE_STATIC_DATA]); */
1873 fprintf (heap_dump_file, "<pinned type=\"other\" bytes=\"%zu\"/>\n", sgen_pin_stats_get_pinned_byte_count (PIN_TYPE_OTHER));
1875 fprintf (heap_dump_file, "<pinned-objects>\n");
1876 for (list = sgen_pin_stats_get_object_list (); list; list = list->next)
1877 dump_object (list->obj, TRUE);
1878 fprintf (heap_dump_file, "</pinned-objects>\n");
1880 sgen_dump_section (nursery_section, "nursery");
1882 major_collector.dump_heap (heap_dump_file);
1884 fprintf (heap_dump_file, "<los>\n");
1885 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next)
1886 dump_object ((MonoObject*)bigobj->data, FALSE);
1887 fprintf (heap_dump_file, "</los>\n");
1889 fprintf (heap_dump_file, "</collection>\n");
1893 sgen_register_moved_object (void *obj, void *destination)
1895 g_assert (mono_profiler_events & MONO_PROFILE_GC_MOVES);
1897 if (moved_objects_idx == MOVED_OBJECTS_NUM) {
1898 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
1899 moved_objects_idx = 0;
1901 moved_objects [moved_objects_idx++] = obj;
1902 moved_objects [moved_objects_idx++] = destination;
1908 static gboolean inited = FALSE;
1913 mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_pre_collection_fragment_clear);
1914 mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_pinning);
1915 mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_scan_remsets);
1916 mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_scan_pinned);
1917 mono_counters_register ("Minor scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_scan_registered_roots);
1918 mono_counters_register ("Minor scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_scan_thread_data);
1919 mono_counters_register ("Minor finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_finish_gray_stack);
1920 mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_minor_fragment_creation);
1922 mono_counters_register ("Major fragment clear", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_pre_collection_fragment_clear);
1923 mono_counters_register ("Major pinning", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_pinning);
1924 mono_counters_register ("Major scan pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_pinned);
1925 mono_counters_register ("Major scan registered roots", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_registered_roots);
1926 mono_counters_register ("Major scan thread data", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_thread_data);
1927 mono_counters_register ("Major scan alloc_pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_alloc_pinned);
1928 mono_counters_register ("Major scan finalized", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_finalized);
1929 mono_counters_register ("Major scan big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_scan_big_objects);
1930 mono_counters_register ("Major finish gray stack", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_finish_gray_stack);
1931 mono_counters_register ("Major free big objects", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_free_bigobjs);
1932 mono_counters_register ("Major LOS sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_los_sweep);
1933 mono_counters_register ("Major sweep", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_sweep);
1934 mono_counters_register ("Major fragment creation", MONO_COUNTER_GC | MONO_COUNTER_LONG | MONO_COUNTER_TIME, &time_major_fragment_creation);
1936 mono_counters_register ("Number of pinned objects", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_pinned_objects);
1938 #ifdef HEAVY_STATISTICS
1939 mono_counters_register ("WBarrier remember pointer", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_add_to_global_remset);
1940 mono_counters_register ("WBarrier set field", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_field);
1941 mono_counters_register ("WBarrier set arrayref", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_arrayref);
1942 mono_counters_register ("WBarrier arrayref copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_arrayref_copy);
1943 mono_counters_register ("WBarrier generic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store);
1944 mono_counters_register ("WBarrier generic atomic store called", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_generic_store_atomic);
1945 mono_counters_register ("WBarrier set root", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_set_root);
1946 mono_counters_register ("WBarrier value copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_value_copy);
1947 mono_counters_register ("WBarrier object copy", MONO_COUNTER_GC | MONO_COUNTER_INT, &stat_wbarrier_object_copy);
1949 mono_counters_register ("# objects allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_alloced_degraded);
1950 mono_counters_register ("bytes allocated degraded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_bytes_alloced_degraded);
1952 mono_counters_register ("# copy_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_nursery);
1953 mono_counters_register ("# objects copied (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_nursery);
1954 mono_counters_register ("# copy_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_copy_object_called_major);
1955 mono_counters_register ("# objects copied (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_objects_copied_major);
1957 mono_counters_register ("# scan_object() called (nursery)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_nursery);
1958 mono_counters_register ("# scan_object() called (major)", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_scan_object_called_major);
1960 mono_counters_register ("Slots allocated in vain", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_slots_allocated_in_vain);
1962 mono_counters_register ("# nursery copy_object() failed from space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_from_space);
1963 mono_counters_register ("# nursery copy_object() failed forwarded", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_forwarded);
1964 mono_counters_register ("# nursery copy_object() failed pinned", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_pinned);
1965 mono_counters_register ("# nursery copy_object() failed to space", MONO_COUNTER_GC | MONO_COUNTER_LONG, &stat_nursery_copy_object_failed_to_space);
1967 sgen_nursery_allocator_init_heavy_stats ();
1968 sgen_alloc_init_heavy_stats ();
1976 reset_pinned_from_failed_allocation (void)
1978 bytes_pinned_from_failed_allocation = 0;
1982 sgen_set_pinned_from_failed_allocation (mword objsize)
1984 bytes_pinned_from_failed_allocation += objsize;
1988 sgen_collection_is_concurrent (void)
1990 switch (current_collection_generation) {
1991 case GENERATION_NURSERY:
1993 case GENERATION_OLD:
1994 return concurrent_collection_in_progress;
1996 g_error ("Invalid current generation %d", current_collection_generation);
2001 sgen_concurrent_collection_in_progress (void)
2003 return concurrent_collection_in_progress;
2010 } FinishRememberedSetScanJobData;
2013 job_finish_remembered_set_scan (WorkerData *worker_data, void *job_data_untyped)
2015 FinishRememberedSetScanJobData *job_data = job_data_untyped;
2017 remset.finish_scan_remsets (job_data->heap_start, job_data->heap_end, sgen_workers_get_job_gray_queue (worker_data));
2018 sgen_free_internal_dynamic (job_data, sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2023 CopyOrMarkObjectFunc copy_or_mark_func;
2024 ScanObjectFunc scan_func;
2028 } ScanFromRegisteredRootsJobData;
2031 job_scan_from_registered_roots (WorkerData *worker_data, void *job_data_untyped)
2033 ScanFromRegisteredRootsJobData *job_data = job_data_untyped;
2034 ScanCopyContext ctx = { job_data->scan_func, job_data->copy_or_mark_func,
2035 sgen_workers_get_job_gray_queue (worker_data) };
2037 scan_from_registered_roots (job_data->heap_start, job_data->heap_end, job_data->root_type, ctx);
2038 sgen_free_internal_dynamic (job_data, sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2045 } ScanThreadDataJobData;
2048 job_scan_thread_data (WorkerData *worker_data, void *job_data_untyped)
2050 ScanThreadDataJobData *job_data = job_data_untyped;
2052 scan_thread_data (job_data->heap_start, job_data->heap_end, TRUE,
2053 sgen_workers_get_job_gray_queue (worker_data));
2054 sgen_free_internal_dynamic (job_data, sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2059 FinalizeReadyEntry *list;
2060 } ScanFinalizerEntriesJobData;
2063 job_scan_finalizer_entries (WorkerData *worker_data, void *job_data_untyped)
2065 ScanFinalizerEntriesJobData *job_data = job_data_untyped;
2066 ScanCopyContext ctx = { NULL, current_object_ops.copy_or_mark_object, sgen_workers_get_job_gray_queue (worker_data) };
2068 scan_finalizer_entries (job_data->list, ctx);
2069 sgen_free_internal_dynamic (job_data, sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA);
2073 job_scan_major_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2075 g_assert (concurrent_collection_in_progress);
2076 major_collector.scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2080 job_scan_los_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped)
2082 g_assert (concurrent_collection_in_progress);
2083 sgen_los_scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data));
2087 verify_scan_starts (char *start, char *end)
2091 for (i = 0; i < nursery_section->num_scan_start; ++i) {
2092 char *addr = nursery_section->scan_starts [i];
2093 if (addr > start && addr < end)
2094 SGEN_LOG (1, "NFC-BAD SCAN START [%zu] %p for obj [%p %p]", i, addr, start, end);
2099 verify_nursery (void)
2101 char *start, *end, *cur, *hole_start;
2103 if (!do_verify_nursery)
2106 /*This cleans up unused fragments */
2107 sgen_nursery_allocator_prepare_for_pinning ();
2109 hole_start = start = cur = sgen_get_nursery_start ();
2110 end = sgen_get_nursery_end ();
2115 if (!*(void**)cur) {
2116 cur += sizeof (void*);
2120 if (object_is_forwarded (cur))
2121 SGEN_LOG (1, "FORWARDED OBJ %p", cur);
2122 else if (object_is_pinned (cur))
2123 SGEN_LOG (1, "PINNED OBJ %p", cur);
2125 ss = safe_object_get_size ((MonoObject*)cur);
2126 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2127 verify_scan_starts (cur, cur + size);
2128 if (do_dump_nursery_content) {
2129 if (cur > hole_start)
2130 SGEN_LOG (1, "HOLE [%p %p %d]", hole_start, cur, (int)(cur - hole_start));
2131 SGEN_LOG (1, "OBJ [%p %p %d %d %s %d]", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ());
2139 * Checks that no objects in the nursery are fowarded or pinned. This
2140 * is a precondition to restarting the mutator while doing a
2141 * concurrent collection. Note that we don't clear fragments because
2142 * we depend on that having happened earlier.
2145 check_nursery_is_clean (void)
2147 char *start, *end, *cur;
2149 start = cur = sgen_get_nursery_start ();
2150 end = sgen_get_nursery_end ();
2155 if (!*(void**)cur) {
2156 cur += sizeof (void*);
2160 g_assert (!object_is_forwarded (cur));
2161 g_assert (!object_is_pinned (cur));
2163 ss = safe_object_get_size ((MonoObject*)cur);
2164 size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur));
2165 verify_scan_starts (cur, cur + size);
2172 init_gray_queue (void)
2174 if (sgen_collection_is_concurrent ()) {
2175 sgen_workers_init_distribute_gray_queue ();
2176 sgen_gray_object_queue_init_with_alloc_prepare (&gray_queue, NULL,
2177 gray_queue_redirect, sgen_workers_get_distribute_section_gray_queue ());
2179 sgen_gray_object_queue_init (&gray_queue, NULL);
2184 pin_stage_object_callback (char *obj, size_t size, void *data)
2186 sgen_pin_stage_ptr (obj);
2187 /* FIXME: do pin stats if enabled */
2191 * Collect objects in the nursery. Returns whether to trigger a major
2195 collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark)
2197 gboolean needs_major;
2198 size_t max_garbage_amount;
2200 FinishRememberedSetScanJobData *frssjd;
2201 ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2202 ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2203 ScanThreadDataJobData *stdjd;
2204 mword fragment_total;
2205 ScanCopyContext ctx;
2209 if (disable_minor_collections)
2212 TV_GETTIME (last_minor_collection_start_tv);
2213 atv = last_minor_collection_start_tv;
2215 MONO_GC_BEGIN (GENERATION_NURSERY);
2216 binary_protocol_collection_begin (gc_stats.minor_gc_count, GENERATION_NURSERY);
2220 #ifndef DISABLE_PERFCOUNTERS
2221 mono_perfcounters->gc_collections0++;
2224 current_collection_generation = GENERATION_NURSERY;
2225 current_object_ops = sgen_minor_collector.serial_ops;
2227 reset_pinned_from_failed_allocation ();
2229 check_scan_starts ();
2231 sgen_nursery_alloc_prepare_for_minor ();
2235 nursery_next = sgen_nursery_alloc_get_upper_alloc_bound ();
2236 /* FIXME: optimize later to use the higher address where an object can be present */
2237 nursery_next = MAX (nursery_next, sgen_get_nursery_end ());
2239 SGEN_LOG (1, "Start nursery collection %d %p-%p, size: %d", gc_stats.minor_gc_count, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ()));
2240 max_garbage_amount = nursery_next - sgen_get_nursery_start ();
2241 g_assert (nursery_section->size >= max_garbage_amount);
2243 /* world must be stopped already */
2245 time_minor_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2247 if (xdomain_checks) {
2248 sgen_clear_nursery_fragments ();
2249 sgen_check_for_xdomain_refs ();
2252 nursery_section->next_data = nursery_next;
2254 major_collector.start_nursery_collection ();
2256 sgen_memgov_minor_collection_start ();
2260 gc_stats.minor_gc_count ++;
2262 MONO_GC_CHECKPOINT_1 (GENERATION_NURSERY);
2264 sgen_process_fin_stage_entries ();
2265 sgen_process_dislink_stage_entries ();
2267 MONO_GC_CHECKPOINT_2 (GENERATION_NURSERY);
2269 /* pin from pinned handles */
2270 sgen_init_pinning ();
2271 mono_profiler_gc_event (MONO_GC_EVENT_MARK_START, 0);
2272 pin_from_roots (sgen_get_nursery_start (), nursery_next, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2273 /* pin cemented objects */
2274 sgen_cement_iterate (pin_stage_object_callback, NULL);
2275 /* identify pinned objects */
2276 sgen_optimize_pin_queue ();
2277 sgen_pinning_setup_section (nursery_section);
2278 ctx.scan_func = NULL;
2279 ctx.copy_func = NULL;
2280 ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2281 pin_objects_in_nursery (ctx);
2282 sgen_pinning_trim_queue_to_section (nursery_section);
2285 time_minor_pinning += TV_ELAPSED (btv, atv);
2286 SGEN_LOG (2, "Finding pinned pointers: %zd in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv));
2287 SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ());
2289 MONO_GC_CHECKPOINT_3 (GENERATION_NURSERY);
2291 if (whole_heap_check_before_collection) {
2292 sgen_clear_nursery_fragments ();
2293 sgen_check_whole_heap (finish_up_concurrent_mark);
2295 if (consistency_check_at_minor_collection)
2296 sgen_check_consistency ();
2298 sgen_workers_start_all_workers ();
2299 sgen_workers_start_marking ();
2301 frssjd = sgen_alloc_internal_dynamic (sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2302 frssjd->heap_start = sgen_get_nursery_start ();
2303 frssjd->heap_end = nursery_next;
2304 sgen_workers_enqueue_job (job_finish_remembered_set_scan, frssjd);
2306 /* we don't have complete write barrier yet, so we scan all the old generation sections */
2308 time_minor_scan_remsets += TV_ELAPSED (atv, btv);
2309 SGEN_LOG (2, "Old generation scan: %d usecs", TV_ELAPSED (atv, btv));
2311 MONO_GC_CHECKPOINT_4 (GENERATION_NURSERY);
2313 ctx.scan_func = current_object_ops.scan_object;
2314 ctx.copy_func = NULL;
2315 ctx.queue = &gray_queue;
2316 sgen_drain_gray_stack (-1, ctx);
2318 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2319 report_registered_roots ();
2320 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2321 report_finalizer_roots ();
2323 time_minor_scan_pinned += TV_ELAPSED (btv, atv);
2325 MONO_GC_CHECKPOINT_5 (GENERATION_NURSERY);
2327 /* registered roots, this includes static fields */
2328 scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2329 scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2330 scrrjd_normal->scan_func = current_object_ops.scan_object;
2331 scrrjd_normal->heap_start = sgen_get_nursery_start ();
2332 scrrjd_normal->heap_end = nursery_next;
2333 scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2334 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2336 scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2337 scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2338 scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2339 scrrjd_wbarrier->heap_start = sgen_get_nursery_start ();
2340 scrrjd_wbarrier->heap_end = nursery_next;
2341 scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2342 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2345 time_minor_scan_registered_roots += TV_ELAPSED (atv, btv);
2347 MONO_GC_CHECKPOINT_6 (GENERATION_NURSERY);
2350 stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2351 stdjd->heap_start = sgen_get_nursery_start ();
2352 stdjd->heap_end = nursery_next;
2353 sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2356 time_minor_scan_thread_data += TV_ELAPSED (btv, atv);
2359 MONO_GC_CHECKPOINT_7 (GENERATION_NURSERY);
2361 g_assert (!sgen_collection_is_concurrent ());
2363 /* Scan the list of objects ready for finalization. If */
2364 sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2365 sfejd_fin_ready->list = fin_ready_list;
2366 sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
2368 sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2369 sfejd_critical_fin->list = critical_fin_list;
2370 sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
2372 MONO_GC_CHECKPOINT_8 (GENERATION_NURSERY);
2374 finish_gray_stack (GENERATION_NURSERY, &gray_queue);
2376 time_minor_finish_gray_stack += TV_ELAPSED (btv, atv);
2377 mono_profiler_gc_event (MONO_GC_EVENT_MARK_END, 0);
2379 MONO_GC_CHECKPOINT_9 (GENERATION_NURSERY);
2382 * The (single-threaded) finalization code might have done
2383 * some copying/marking so we can only reset the GC thread's
2384 * worker data here instead of earlier when we joined the
2387 sgen_workers_reset_data ();
2389 if (objects_pinned) {
2390 sgen_optimize_pin_queue ();
2391 sgen_pinning_setup_section (nursery_section);
2394 /* walk the pin_queue, build up the fragment list of free memory, unmark
2395 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2398 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_START, 0);
2399 fragment_total = sgen_build_nursery_fragments (nursery_section,
2400 nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries,
2402 if (!fragment_total)
2405 /* Clear TLABs for all threads */
2406 sgen_clear_tlabs ();
2408 mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0);
2410 time_minor_fragment_creation += TV_ELAPSED (atv, btv);
2411 SGEN_LOG (2, "Fragment creation: %d usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total);
2413 if (consistency_check_at_minor_collection)
2414 sgen_check_major_refs ();
2416 major_collector.finish_nursery_collection ();
2418 TV_GETTIME (last_minor_collection_end_tv);
2419 gc_stats.minor_gc_time += TV_ELAPSED (last_minor_collection_start_tv, last_minor_collection_end_tv);
2422 dump_heap ("minor", gc_stats.minor_gc_count - 1, NULL);
2424 /* prepare the pin queue for the next collection */
2425 sgen_finish_pinning ();
2426 if (fin_ready_list || critical_fin_list) {
2427 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2428 mono_gc_finalize_notify ();
2430 sgen_pin_stats_reset ();
2431 /* clear cemented hash */
2432 sgen_cement_clear_below_threshold ();
2434 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2436 remset.finish_minor_collection ();
2438 check_scan_starts ();
2440 binary_protocol_flush_buffers (FALSE);
2442 sgen_memgov_minor_collection_end ();
2444 /*objects are late pinned because of lack of memory, so a major is a good call*/
2445 needs_major = objects_pinned > 0;
2446 current_collection_generation = -1;
2449 MONO_GC_END (GENERATION_NURSERY);
2450 binary_protocol_collection_end (gc_stats.minor_gc_count - 1, GENERATION_NURSERY);
2452 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2453 sgen_check_nursery_objects_pinned (unpin_queue != NULL);
2459 scan_nursery_objects_callback (char *obj, size_t size, ScanCopyContext *ctx)
2461 ctx->scan_func (obj, sgen_obj_get_descriptor (obj), ctx->queue);
2465 scan_nursery_objects (ScanCopyContext ctx)
2467 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
2468 (IterateObjectCallbackFunc)scan_nursery_objects_callback, (void*)&ctx, FALSE);
2472 major_copy_or_mark_from_roots (size_t *old_next_pin_slot, gboolean finish_up_concurrent_mark, gboolean scan_mod_union)
2477 /* FIXME: only use these values for the precise scan
2478 * note that to_space pointers should be excluded anyway...
2480 char *heap_start = NULL;
2481 char *heap_end = (char*)-1;
2482 gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS;
2483 GCRootReport root_report = { 0 };
2484 ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier;
2485 ScanThreadDataJobData *stdjd;
2486 ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin;
2487 ScanCopyContext ctx;
2489 if (concurrent_collection_in_progress) {
2490 /*This cleans up unused fragments */
2491 sgen_nursery_allocator_prepare_for_pinning ();
2493 if (do_concurrent_checks)
2494 check_nursery_is_clean ();
2496 /* The concurrent collector doesn't touch the nursery. */
2497 sgen_nursery_alloc_prepare_for_major ();
2504 /* Pinning depends on this */
2505 sgen_clear_nursery_fragments ();
2507 if (whole_heap_check_before_collection)
2508 sgen_check_whole_heap (finish_up_concurrent_mark);
2511 time_major_pre_collection_fragment_clear += TV_ELAPSED (atv, btv);
2513 if (!sgen_collection_is_concurrent ())
2514 nursery_section->next_data = sgen_get_nursery_end ();
2515 /* we should also coalesce scanning from sections close to each other
2516 * and deal with pointers outside of the sections later.
2520 *major_collector.have_swept = FALSE;
2522 if (xdomain_checks) {
2523 sgen_clear_nursery_fragments ();
2524 sgen_check_for_xdomain_refs ();
2527 if (!concurrent_collection_in_progress) {
2528 /* Remsets are not useful for a major collection */
2529 remset.prepare_for_major_collection ();
2532 sgen_process_fin_stage_entries ();
2533 sgen_process_dislink_stage_entries ();
2536 sgen_init_pinning ();
2537 SGEN_LOG (6, "Collecting pinned addresses");
2538 pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, WORKERS_DISTRIBUTE_GRAY_QUEUE);
2540 if (!concurrent_collection_in_progress || finish_up_concurrent_mark) {
2541 if (major_collector.is_concurrent) {
2543 * The concurrent major collector cannot evict
2544 * yet, so we need to pin cemented objects to
2545 * not break some asserts.
2547 * FIXME: We could evict now!
2549 sgen_cement_iterate (pin_stage_object_callback, NULL);
2552 if (!concurrent_collection_in_progress)
2553 sgen_cement_reset ();
2556 sgen_optimize_pin_queue ();
2559 * The concurrent collector doesn't move objects, neither on
2560 * the major heap nor in the nursery, so we can mark even
2561 * before pinning has finished. For the non-concurrent
2562 * collector we start the workers after pinning.
2564 if (concurrent_collection_in_progress) {
2565 sgen_workers_start_all_workers ();
2566 sgen_workers_start_marking ();
2570 * pin_queue now contains all candidate pointers, sorted and
2571 * uniqued. We must do two passes now to figure out which
2572 * objects are pinned.
2574 * The first is to find within the pin_queue the area for each
2575 * section. This requires that the pin_queue be sorted. We
2576 * also process the LOS objects and pinned chunks here.
2578 * The second, destructive, pass is to reduce the section
2579 * areas to pointers to the actually pinned objects.
2581 SGEN_LOG (6, "Pinning from sections");
2582 /* first pass for the sections */
2583 sgen_find_section_pin_queue_start_end (nursery_section);
2584 major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2585 /* identify possible pointers to the insize of large objects */
2586 SGEN_LOG (6, "Pinning from large objects");
2587 for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) {
2589 if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + sgen_los_object_size (bigobj), &dummy)) {
2590 binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (((MonoObject*)(bigobj->data))));
2592 #ifdef ENABLE_DTRACE
2593 if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) {
2594 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (bigobj->data);
2595 MONO_GC_OBJ_PINNED ((mword)bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD);
2599 if (sgen_los_object_is_pinned (bigobj->data)) {
2600 g_assert (finish_up_concurrent_mark);
2603 sgen_los_pin_object (bigobj->data);
2604 if (SGEN_OBJECT_HAS_REFERENCES (bigobj->data))
2605 GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data, sgen_obj_get_descriptor (bigobj->data));
2606 if (G_UNLIKELY (do_pin_stats))
2607 sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data));
2608 SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data, safe_name (bigobj->data), (unsigned long)sgen_los_object_size (bigobj));
2611 add_profile_gc_root (&root_report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0);
2615 notify_gc_roots (&root_report);
2616 /* second pass for the sections */
2617 ctx.scan_func = concurrent_collection_in_progress ? current_object_ops.scan_object : NULL;
2618 ctx.copy_func = NULL;
2619 ctx.queue = WORKERS_DISTRIBUTE_GRAY_QUEUE;
2622 * Concurrent mark never follows references into the nursery.
2623 * In the start and finish pauses we must scan live nursery
2624 * objects, though. We could simply scan all nursery objects,
2625 * but that would be conservative. The easiest way is to do a
2626 * nursery collection, which copies all live nursery objects
2627 * (except pinned ones, with the simple nursery) to the major
2628 * heap. Scanning the mod union table later will then scan
2629 * those promoted objects, provided they're reachable. Pinned
2630 * objects in the nursery - which we can trivially find in the
2631 * pinning queue - are treated as roots in the mark pauses.
2633 * The split nursery complicates the latter part because
2634 * non-pinned objects can survive in the nursery. That's why
2635 * we need to do a full front-to-back scan of the nursery,
2636 * marking all objects.
2638 * Non-concurrent mark evacuates from the nursery, so it's
2639 * sufficient to just scan pinned nursery objects.
2641 if (concurrent_collection_in_progress && sgen_minor_collector.is_split) {
2642 scan_nursery_objects (ctx);
2644 pin_objects_in_nursery (ctx);
2645 if (check_nursery_objects_pinned && !sgen_minor_collector.is_split)
2646 sgen_check_nursery_objects_pinned (!concurrent_collection_in_progress || finish_up_concurrent_mark);
2649 major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE);
2650 if (old_next_pin_slot)
2651 *old_next_pin_slot = sgen_get_pinned_count ();
2654 time_major_pinning += TV_ELAPSED (atv, btv);
2655 SGEN_LOG (2, "Finding pinned pointers: %zd in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv));
2656 SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ());
2658 major_collector.init_to_space ();
2660 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2661 main_gc_thread = mono_native_thread_self ();
2664 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2665 report_registered_roots ();
2667 time_major_scan_pinned += TV_ELAPSED (btv, atv);
2669 /* registered roots, this includes static fields */
2670 scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2671 scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2672 scrrjd_normal->scan_func = current_object_ops.scan_object;
2673 scrrjd_normal->heap_start = heap_start;
2674 scrrjd_normal->heap_end = heap_end;
2675 scrrjd_normal->root_type = ROOT_TYPE_NORMAL;
2676 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal);
2678 scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2679 scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object;
2680 scrrjd_wbarrier->scan_func = current_object_ops.scan_object;
2681 scrrjd_wbarrier->heap_start = heap_start;
2682 scrrjd_wbarrier->heap_end = heap_end;
2683 scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER;
2684 sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier);
2687 time_major_scan_registered_roots += TV_ELAPSED (atv, btv);
2690 stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2691 stdjd->heap_start = heap_start;
2692 stdjd->heap_end = heap_end;
2693 sgen_workers_enqueue_job (job_scan_thread_data, stdjd);
2696 time_major_scan_thread_data += TV_ELAPSED (btv, atv);
2699 time_major_scan_alloc_pinned += TV_ELAPSED (atv, btv);
2701 if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS)
2702 report_finalizer_roots ();
2704 /* scan the list of objects ready for finalization */
2705 sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2706 sfejd_fin_ready->list = fin_ready_list;
2707 sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready);
2709 sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE);
2710 sfejd_critical_fin->list = critical_fin_list;
2711 sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin);
2713 if (scan_mod_union) {
2714 g_assert (finish_up_concurrent_mark);
2716 /* Mod union card table */
2717 sgen_workers_enqueue_job (job_scan_major_mod_union_cardtable, NULL);
2718 sgen_workers_enqueue_job (job_scan_los_mod_union_cardtable, NULL);
2722 time_major_scan_finalized += TV_ELAPSED (btv, atv);
2723 SGEN_LOG (2, "Root scan: %d usecs", TV_ELAPSED (btv, atv));
2726 time_major_scan_big_objects += TV_ELAPSED (atv, btv);
2728 if (concurrent_collection_in_progress) {
2729 /* prepare the pin queue for the next collection */
2730 sgen_finish_pinning ();
2732 sgen_pin_stats_reset ();
2734 if (do_concurrent_checks)
2735 check_nursery_is_clean ();
2740 major_start_collection (gboolean concurrent, size_t *old_next_pin_slot)
2742 MONO_GC_BEGIN (GENERATION_OLD);
2743 binary_protocol_collection_begin (gc_stats.major_gc_count, GENERATION_OLD);
2745 current_collection_generation = GENERATION_OLD;
2746 #ifndef DISABLE_PERFCOUNTERS
2747 mono_perfcounters->gc_collections1++;
2750 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2753 g_assert (major_collector.is_concurrent);
2754 concurrent_collection_in_progress = TRUE;
2756 sgen_cement_concurrent_start ();
2758 current_object_ops = major_collector.major_concurrent_ops;
2760 current_object_ops = major_collector.major_ops;
2763 reset_pinned_from_failed_allocation ();
2765 sgen_memgov_major_collection_start ();
2767 //count_ref_nonref_objs ();
2768 //consistency_check ();
2770 check_scan_starts ();
2773 SGEN_LOG (1, "Start major collection %d", gc_stats.major_gc_count);
2774 gc_stats.major_gc_count ++;
2776 if (major_collector.start_major_collection)
2777 major_collector.start_major_collection ();
2779 major_copy_or_mark_from_roots (old_next_pin_slot, FALSE, FALSE);
2783 wait_for_workers_to_finish (void)
2785 while (!sgen_workers_all_done ())
2792 if (concurrent_collection_in_progress) {
2793 gray_queue_redirect (&gray_queue);
2794 sgen_workers_join ();
2797 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2799 #ifdef SGEN_DEBUG_INTERNAL_ALLOC
2800 main_gc_thread = NULL;
2805 major_finish_collection (const char *reason, size_t old_next_pin_slot, gboolean scan_mod_union)
2807 LOSObject *bigobj, *prevbo;
2813 if (concurrent_collection_in_progress)
2816 if (concurrent_collection_in_progress) {
2817 current_object_ops = major_collector.major_concurrent_ops;
2819 major_copy_or_mark_from_roots (NULL, TRUE, scan_mod_union);
2822 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2824 if (do_concurrent_checks)
2825 check_nursery_is_clean ();
2827 current_object_ops = major_collector.major_ops;
2831 * The workers have stopped so we need to finish gray queue
2832 * work that might result from finalization in the main GC
2833 * thread. Redirection must therefore be turned off.
2835 sgen_gray_object_queue_disable_alloc_prepare (&gray_queue);
2836 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2838 /* all the objects in the heap */
2839 finish_gray_stack (GENERATION_OLD, &gray_queue);
2841 time_major_finish_gray_stack += TV_ELAPSED (btv, atv);
2844 * The (single-threaded) finalization code might have done
2845 * some copying/marking so we can only reset the GC thread's
2846 * worker data here instead of earlier when we joined the
2849 sgen_workers_reset_data ();
2851 if (objects_pinned) {
2852 g_assert (!concurrent_collection_in_progress);
2854 /*This is slow, but we just OOM'd*/
2855 sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot);
2856 sgen_optimize_pin_queue ();
2857 sgen_find_section_pin_queue_start_end (nursery_section);
2861 reset_heap_boundaries ();
2862 sgen_update_heap_boundaries ((mword)sgen_get_nursery_start (), (mword)sgen_get_nursery_end ());
2864 if (check_mark_bits_after_major_collection)
2865 sgen_check_major_heap_marked ();
2867 MONO_GC_SWEEP_BEGIN (GENERATION_OLD, !major_collector.sweeps_lazily);
2869 /* sweep the big objects list */
2871 for (bigobj = los_object_list; bigobj;) {
2872 g_assert (!object_is_pinned (bigobj->data));
2873 if (sgen_los_object_is_pinned (bigobj->data)) {
2874 sgen_los_unpin_object (bigobj->data);
2875 sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj));
2878 /* not referenced anywhere, so we can free it */
2880 prevbo->next = bigobj->next;
2882 los_object_list = bigobj->next;
2884 bigobj = bigobj->next;
2885 sgen_los_free_object (to_free);
2889 bigobj = bigobj->next;
2893 time_major_free_bigobjs += TV_ELAPSED (atv, btv);
2898 time_major_los_sweep += TV_ELAPSED (btv, atv);
2900 major_collector.sweep ();
2902 MONO_GC_SWEEP_END (GENERATION_OLD, !major_collector.sweeps_lazily);
2905 time_major_sweep += TV_ELAPSED (atv, btv);
2907 if (!concurrent_collection_in_progress) {
2908 /* walk the pin_queue, build up the fragment list of free memory, unmark
2909 * pinned objects as we go, memzero() the empty fragments so they are ready for the
2912 if (!sgen_build_nursery_fragments (nursery_section, nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries, NULL))
2915 /* prepare the pin queue for the next collection */
2916 sgen_finish_pinning ();
2918 /* Clear TLABs for all threads */
2919 sgen_clear_tlabs ();
2921 sgen_pin_stats_reset ();
2924 if (concurrent_collection_in_progress)
2925 sgen_cement_concurrent_finish ();
2926 sgen_cement_clear_below_threshold ();
2929 time_major_fragment_creation += TV_ELAPSED (btv, atv);
2932 dump_heap ("major", gc_stats.major_gc_count - 1, reason);
2934 if (fin_ready_list || critical_fin_list) {
2935 SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers);
2936 mono_gc_finalize_notify ();
2939 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
2941 sgen_memgov_major_collection_end ();
2942 current_collection_generation = -1;
2944 major_collector.finish_major_collection ();
2946 g_assert (sgen_section_gray_queue_is_empty (sgen_workers_get_distribute_section_gray_queue ()));
2948 if (concurrent_collection_in_progress)
2949 concurrent_collection_in_progress = FALSE;
2951 check_scan_starts ();
2953 binary_protocol_flush_buffers (FALSE);
2955 //consistency_check ();
2957 MONO_GC_END (GENERATION_OLD);
2958 binary_protocol_collection_end (gc_stats.major_gc_count - 1, GENERATION_OLD);
2962 major_do_collection (const char *reason)
2964 TV_DECLARE (time_start);
2965 TV_DECLARE (time_end);
2966 size_t old_next_pin_slot;
2968 if (disable_major_collections)
2971 if (major_collector.get_and_reset_num_major_objects_marked) {
2972 long long num_marked = major_collector.get_and_reset_num_major_objects_marked ();
2973 g_assert (!num_marked);
2976 /* world must be stopped already */
2977 TV_GETTIME (time_start);
2979 major_start_collection (FALSE, &old_next_pin_slot);
2980 major_finish_collection (reason, old_next_pin_slot, FALSE);
2982 TV_GETTIME (time_end);
2983 gc_stats.major_gc_time += TV_ELAPSED (time_start, time_end);
2985 /* FIXME: also report this to the user, preferably in gc-end. */
2986 if (major_collector.get_and_reset_num_major_objects_marked)
2987 major_collector.get_and_reset_num_major_objects_marked ();
2989 return bytes_pinned_from_failed_allocation > 0;
2993 major_start_concurrent_collection (const char *reason)
2995 TV_DECLARE (time_start);
2996 TV_DECLARE (time_end);
2997 long long num_objects_marked;
2999 if (disable_major_collections)
3002 TV_GETTIME (time_start);
3003 SGEN_TV_GETTIME (time_major_conc_collection_start);
3005 num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3006 g_assert (num_objects_marked == 0);
3008 MONO_GC_CONCURRENT_START_BEGIN (GENERATION_OLD);
3009 binary_protocol_concurrent_start ();
3011 // FIXME: store reason and pass it when finishing
3012 major_start_collection (TRUE, NULL);
3014 gray_queue_redirect (&gray_queue);
3015 sgen_workers_wait_for_jobs ();
3017 num_objects_marked = major_collector.get_and_reset_num_major_objects_marked ();
3018 MONO_GC_CONCURRENT_START_END (GENERATION_OLD, num_objects_marked);
3020 TV_GETTIME (time_end);
3021 gc_stats.major_gc_time += TV_ELAPSED (time_start, time_end);
3023 current_collection_generation = -1;
3027 major_update_or_finish_concurrent_collection (gboolean force_finish)
3029 TV_DECLARE (total_start);
3030 TV_DECLARE (total_end);
3031 SgenGrayQueue unpin_queue;
3032 memset (&unpin_queue, 0, sizeof (unpin_queue));
3034 TV_GETTIME (total_start);
3036 MONO_GC_CONCURRENT_UPDATE_FINISH_BEGIN (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3037 binary_protocol_concurrent_update_finish ();
3039 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3041 if (!force_finish && !sgen_workers_all_done ()) {
3042 major_collector.update_cardtable_mod_union ();
3043 sgen_los_update_cardtable_mod_union ();
3045 MONO_GC_CONCURRENT_UPDATE_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3047 TV_GETTIME (total_end);
3048 gc_stats.major_gc_time += TV_ELAPSED (total_start, total_end);
3054 * The major collector can add global remsets which are processed in the finishing
3055 * nursery collection, below. That implies that the workers must have finished
3056 * marking before the nursery collection is allowed to run, otherwise we might miss
3059 wait_for_workers_to_finish ();
3061 SGEN_TV_GETTIME (time_major_conc_collection_end);
3062 gc_stats.major_gc_time_concurrent += SGEN_TV_ELAPSED (time_major_conc_collection_start, time_major_conc_collection_end);
3064 major_collector.update_cardtable_mod_union ();
3065 sgen_los_update_cardtable_mod_union ();
3067 collect_nursery (&unpin_queue, TRUE);
3069 if (mod_union_consistency_check)
3070 sgen_check_mod_union_consistency ();
3072 current_collection_generation = GENERATION_OLD;
3073 major_finish_collection ("finishing", -1, TRUE);
3075 if (whole_heap_check_before_collection)
3076 sgen_check_whole_heap (FALSE);
3078 unpin_objects_from_queue (&unpin_queue);
3079 sgen_gray_object_queue_deinit (&unpin_queue);
3081 MONO_GC_CONCURRENT_FINISH_END (GENERATION_OLD, major_collector.get_and_reset_num_major_objects_marked ());
3083 TV_GETTIME (total_end);
3084 gc_stats.major_gc_time += TV_ELAPSED (total_start, total_end) - TV_ELAPSED (last_minor_collection_start_tv, last_minor_collection_end_tv);
3086 current_collection_generation = -1;
3092 * Ensure an allocation request for @size will succeed by freeing enough memory.
3094 * LOCKING: The GC lock MUST be held.
3097 sgen_ensure_free_space (size_t size)
3099 int generation_to_collect = -1;
3100 const char *reason = NULL;
3103 if (size > SGEN_MAX_SMALL_OBJ_SIZE) {
3104 if (sgen_need_major_collection (size)) {
3105 reason = "LOS overflow";
3106 generation_to_collect = GENERATION_OLD;
3109 if (degraded_mode) {
3110 if (sgen_need_major_collection (size)) {
3111 reason = "Degraded mode overflow";
3112 generation_to_collect = GENERATION_OLD;
3114 } else if (sgen_need_major_collection (size)) {
3115 reason = "Minor allowance";
3116 generation_to_collect = GENERATION_OLD;
3118 generation_to_collect = GENERATION_NURSERY;
3119 reason = "Nursery full";
3123 if (generation_to_collect == -1) {
3124 if (concurrent_collection_in_progress && sgen_workers_all_done ()) {
3125 generation_to_collect = GENERATION_OLD;
3126 reason = "Finish concurrent collection";
3130 if (generation_to_collect == -1)
3132 sgen_perform_collection (size, generation_to_collect, reason, FALSE);
3136 * LOCKING: Assumes the GC lock is held.
3139 sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish)
3141 TV_DECLARE (gc_end);
3142 GGTimingInfo infos [2];
3143 int overflow_generation_to_collect = -1;
3144 int oldest_generation_collected = generation_to_collect;
3145 const char *overflow_reason = NULL;
3147 MONO_GC_REQUESTED (generation_to_collect, requested_size, wait_to_finish ? 1 : 0);
3149 binary_protocol_collection_force (generation_to_collect);
3151 g_assert (generation_to_collect == GENERATION_NURSERY || generation_to_collect == GENERATION_OLD);
3153 memset (infos, 0, sizeof (infos));
3154 mono_profiler_gc_event (MONO_GC_EVENT_START, generation_to_collect);
3156 infos [0].generation = generation_to_collect;
3157 infos [0].reason = reason;
3158 infos [0].is_overflow = FALSE;
3159 TV_GETTIME (infos [0].total_time);
3160 infos [1].generation = -1;
3162 sgen_stop_world (generation_to_collect);
3164 if (concurrent_collection_in_progress) {
3165 if (major_update_or_finish_concurrent_collection (wait_to_finish && generation_to_collect == GENERATION_OLD)) {
3166 oldest_generation_collected = GENERATION_OLD;
3169 if (generation_to_collect == GENERATION_OLD)
3172 if (generation_to_collect == GENERATION_OLD &&
3173 allow_synchronous_major &&
3174 major_collector.want_synchronous_collection &&
3175 *major_collector.want_synchronous_collection) {
3176 wait_to_finish = TRUE;
3180 //FIXME extract overflow reason
3181 if (generation_to_collect == GENERATION_NURSERY) {
3182 if (collect_nursery (NULL, FALSE)) {
3183 overflow_generation_to_collect = GENERATION_OLD;
3184 overflow_reason = "Minor overflow";
3187 if (major_collector.is_concurrent) {
3188 g_assert (!concurrent_collection_in_progress);
3189 if (!wait_to_finish)
3190 collect_nursery (NULL, FALSE);
3193 if (major_collector.is_concurrent && !wait_to_finish) {
3194 major_start_concurrent_collection (reason);
3195 // FIXME: set infos[0] properly
3198 if (major_do_collection (reason)) {
3199 overflow_generation_to_collect = GENERATION_NURSERY;
3200 overflow_reason = "Excessive pinning";
3205 TV_GETTIME (gc_end);
3206 infos [0].total_time = SGEN_TV_ELAPSED (infos [0].total_time, gc_end);
3209 if (!major_collector.is_concurrent && overflow_generation_to_collect != -1) {
3210 mono_profiler_gc_event (MONO_GC_EVENT_START, overflow_generation_to_collect);
3211 infos [1].generation = overflow_generation_to_collect;
3212 infos [1].reason = overflow_reason;
3213 infos [1].is_overflow = TRUE;
3214 infos [1].total_time = gc_end;
3216 if (overflow_generation_to_collect == GENERATION_NURSERY)
3217 collect_nursery (NULL, FALSE);
3219 major_do_collection (overflow_reason);
3221 TV_GETTIME (gc_end);
3222 infos [1].total_time = SGEN_TV_ELAPSED (infos [1].total_time, gc_end);
3224 /* keep events symmetric */
3225 mono_profiler_gc_event (MONO_GC_EVENT_END, overflow_generation_to_collect);
3227 oldest_generation_collected = MAX (oldest_generation_collected, overflow_generation_to_collect);
3230 SGEN_LOG (2, "Heap size: %lu, LOS size: %lu", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage);
3232 /* this also sets the proper pointers for the next allocation */
3233 if (generation_to_collect == GENERATION_NURSERY && !sgen_can_alloc_size (requested_size)) {
3234 /* TypeBuilder and MonoMethod are killing mcs with fragmentation */
3235 SGEN_LOG (1, "nursery collection didn't find enough room for %zd alloc (%zd pinned)", requested_size, sgen_get_pinned_count ());
3236 sgen_dump_pin_queue ();
3241 g_assert (sgen_gray_object_queue_is_empty (&gray_queue));
3243 sgen_restart_world (oldest_generation_collected, infos);
3245 mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect);
3249 * ######################################################################
3250 * ######## Memory allocation from the OS
3251 * ######################################################################
3252 * This section of code deals with getting memory from the OS and
3253 * allocating memory for GC-internal data structures.
3254 * Internal memory can be handled with a freelist for small objects.
3260 G_GNUC_UNUSED static void
3261 report_internal_mem_usage (void)
3263 printf ("Internal memory usage:\n");
3264 sgen_report_internal_mem_usage ();
3265 printf ("Pinned memory usage:\n");
3266 major_collector.report_pinned_memory_usage ();
3270 * ######################################################################
3271 * ######## Finalization support
3272 * ######################################################################
3275 static inline gboolean
3276 sgen_major_is_object_alive (void *object)
3280 /* Oldgen objects can be pinned and forwarded too */
3281 if (SGEN_OBJECT_IS_PINNED (object) || SGEN_OBJECT_IS_FORWARDED (object))
3285 * FIXME: major_collector.is_object_live() also calculates the
3286 * size. Avoid the double calculation.
3288 objsize = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)object));
3289 if (objsize > SGEN_MAX_SMALL_OBJ_SIZE)
3290 return sgen_los_object_is_pinned (object);
3292 return major_collector.is_object_live (object);
3296 * If the object has been forwarded it means it's still referenced from a root.
3297 * If it is pinned it's still alive as well.
3298 * A LOS object is only alive if we have pinned it.
3299 * Return TRUE if @obj is ready to be finalized.
3301 static inline gboolean
3302 sgen_is_object_alive (void *object)
3304 if (ptr_in_nursery (object))
3305 return sgen_nursery_is_object_alive (object);
3307 return sgen_major_is_object_alive (object);
3311 * This function returns true if @object is either alive or it belongs to the old gen
3312 * and we're currently doing a minor collection.
3315 sgen_is_object_alive_for_current_gen (char *object)
3317 if (ptr_in_nursery (object))
3318 return sgen_nursery_is_object_alive (object);
3320 if (current_collection_generation == GENERATION_NURSERY)
3323 return sgen_major_is_object_alive (object);
3327 * This function returns true if @object is either alive and belongs to the
3328 * current collection - major collections are full heap, so old gen objects
3329 * are never alive during a minor collection.
3332 sgen_is_object_alive_and_on_current_collection (char *object)
3334 if (ptr_in_nursery (object))
3335 return sgen_nursery_is_object_alive (object);
3337 if (current_collection_generation == GENERATION_NURSERY)
3340 return sgen_major_is_object_alive (object);
3345 sgen_gc_is_object_ready_for_finalization (void *object)
3347 return !sgen_is_object_alive (object);
3351 has_critical_finalizer (MonoObject *obj)
3355 if (!mono_defaults.critical_finalizer_object)
3358 class = ((MonoVTable*)LOAD_VTABLE (obj))->klass;
3360 return mono_class_has_parent_fast (class, mono_defaults.critical_finalizer_object);
3364 is_finalization_aware (MonoObject *obj)
3366 MonoVTable *vt = ((MonoVTable*)LOAD_VTABLE (obj));
3367 return (vt->gc_bits & SGEN_GC_BIT_FINALIZER_AWARE) == SGEN_GC_BIT_FINALIZER_AWARE;
3371 sgen_queue_finalization_entry (MonoObject *obj)
3373 FinalizeReadyEntry *entry = sgen_alloc_internal (INTERNAL_MEM_FINALIZE_READY_ENTRY);
3374 gboolean critical = has_critical_finalizer (obj);
3375 entry->object = obj;
3377 entry->next = critical_fin_list;
3378 critical_fin_list = entry;
3380 entry->next = fin_ready_list;
3381 fin_ready_list = entry;
3384 if (fin_callbacks.object_queued_for_finalization && is_finalization_aware (obj))
3385 fin_callbacks.object_queued_for_finalization (obj);
3387 #ifdef ENABLE_DTRACE
3388 if (G_UNLIKELY (MONO_GC_FINALIZE_ENQUEUE_ENABLED ())) {
3389 int gen = sgen_ptr_in_nursery (obj) ? GENERATION_NURSERY : GENERATION_OLD;
3390 MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (obj);
3391 MONO_GC_FINALIZE_ENQUEUE ((mword)obj, sgen_safe_object_get_size (obj),
3392 vt->klass->name_space, vt->klass->name, gen, critical);
3398 sgen_object_is_live (void *obj)
3400 return sgen_is_object_alive_and_on_current_collection (obj);
3403 /* LOCKING: requires that the GC lock is held */
3405 null_ephemerons_for_domain (MonoDomain *domain)
3407 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3410 MonoObject *object = (MonoObject*)current->array;
3412 if (object && !object->vtable) {
3413 EphemeronLinkNode *tmp = current;
3416 prev->next = current->next;
3418 ephemeron_list = current->next;
3420 current = current->next;
3421 sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3424 current = current->next;
3429 /* LOCKING: requires that the GC lock is held */
3431 clear_unreachable_ephemerons (ScanCopyContext ctx)
3433 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3434 GrayQueue *queue = ctx.queue;
3435 EphemeronLinkNode *current = ephemeron_list, *prev = NULL;
3437 Ephemeron *cur, *array_end;
3441 char *object = current->array;
3443 if (!sgen_is_object_alive_for_current_gen (object)) {
3444 EphemeronLinkNode *tmp = current;
3446 SGEN_LOG (5, "Dead Ephemeron array at %p", object);
3449 prev->next = current->next;
3451 ephemeron_list = current->next;
3453 current = current->next;
3454 sgen_free_internal (tmp, INTERNAL_MEM_EPHEMERON_LINK);
3459 copy_func ((void**)&object, queue);
3460 current->array = object;
3462 SGEN_LOG (5, "Clearing unreachable entries for ephemeron array at %p", object);
3464 array = (MonoArray*)object;
3465 cur = mono_array_addr (array, Ephemeron, 0);
3466 array_end = cur + mono_array_length_fast (array);
3467 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3469 for (; cur < array_end; ++cur) {
3470 char *key = (char*)cur->key;
3472 if (!key || key == tombstone)
3475 SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3476 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3477 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3479 if (!sgen_is_object_alive_for_current_gen (key)) {
3480 cur->key = tombstone;
3486 current = current->next;
3491 LOCKING: requires that the GC lock is held
3493 Limitations: We scan all ephemerons on every collection since the current design doesn't allow for a simple nursery/mature split.
3496 mark_ephemerons_in_range (ScanCopyContext ctx)
3498 CopyOrMarkObjectFunc copy_func = ctx.copy_func;
3499 GrayQueue *queue = ctx.queue;
3500 int nothing_marked = 1;
3501 EphemeronLinkNode *current = ephemeron_list;
3503 Ephemeron *cur, *array_end;
3506 for (current = ephemeron_list; current; current = current->next) {
3507 char *object = current->array;
3508 SGEN_LOG (5, "Ephemeron array at %p", object);
3510 /*It has to be alive*/
3511 if (!sgen_is_object_alive_for_current_gen (object)) {
3512 SGEN_LOG (5, "\tnot reachable");
3516 copy_func ((void**)&object, queue);
3518 array = (MonoArray*)object;
3519 cur = mono_array_addr (array, Ephemeron, 0);
3520 array_end = cur + mono_array_length_fast (array);
3521 tombstone = (char*)((MonoVTable*)LOAD_VTABLE (object))->domain->ephemeron_tombstone;
3523 for (; cur < array_end; ++cur) {
3524 char *key = cur->key;
3526 if (!key || key == tombstone)
3529 SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0),
3530 key, sgen_is_object_alive_for_current_gen (key) ? "reachable" : "unreachable",
3531 cur->value, cur->value && sgen_is_object_alive_for_current_gen (cur->value) ? "reachable" : "unreachable");
3533 if (sgen_is_object_alive_for_current_gen (key)) {
3534 char *value = cur->value;
3536 copy_func ((void**)&cur->key, queue);
3538 if (!sgen_is_object_alive_for_current_gen (value))
3540 copy_func ((void**)&cur->value, queue);
3546 SGEN_LOG (5, "Ephemeron run finished. Is it done %d", nothing_marked);
3547 return nothing_marked;
3551 mono_gc_invoke_finalizers (void)
3553 FinalizeReadyEntry *entry = NULL;
3554 gboolean entry_is_critical = FALSE;
3557 /* FIXME: batch to reduce lock contention */
3558 while (fin_ready_list || critical_fin_list) {
3562 FinalizeReadyEntry **list = entry_is_critical ? &critical_fin_list : &fin_ready_list;
3564 /* We have finalized entry in the last
3565 interation, now we need to remove it from
3568 *list = entry->next;
3570 FinalizeReadyEntry *e = *list;
3571 while (e->next != entry)
3573 e->next = entry->next;
3575 sgen_free_internal (entry, INTERNAL_MEM_FINALIZE_READY_ENTRY);
3579 /* Now look for the first non-null entry. */
3580 for (entry = fin_ready_list; entry && !entry->object; entry = entry->next)
3583 entry_is_critical = FALSE;
3585 entry_is_critical = TRUE;
3586 for (entry = critical_fin_list; entry && !entry->object; entry = entry->next)
3591 g_assert (entry->object);
3592 num_ready_finalizers--;
3593 obj = entry->object;
3594 entry->object = NULL;
3595 SGEN_LOG (7, "Finalizing object %p (%s)", obj, safe_name (obj));
3603 g_assert (entry->object == NULL);
3605 /* the object is on the stack so it is pinned */
3606 /*g_print ("Calling finalizer for object: %p (%s)\n", entry->object, safe_name (entry->object));*/
3607 mono_gc_run_finalize (obj, NULL);
3614 mono_gc_pending_finalizers (void)
3616 return fin_ready_list || critical_fin_list;
3620 * ######################################################################
3621 * ######## registered roots support
3622 * ######################################################################
3626 * We do not coalesce roots.
3629 mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_type)
3631 RootRecord new_root;
3634 for (i = 0; i < ROOT_TYPE_NUM; ++i) {
3635 RootRecord *root = sgen_hash_table_lookup (&roots_hash [i], start);
3636 /* we allow changing the size and the descriptor (for thread statics etc) */
3638 size_t old_size = root->end_root - start;
3639 root->end_root = start + size;
3640 g_assert (((root->root_desc != 0) && (descr != NULL)) ||
3641 ((root->root_desc == 0) && (descr == NULL)));
3642 root->root_desc = (mword)descr;
3644 roots_size -= old_size;
3650 new_root.end_root = start + size;
3651 new_root.root_desc = (mword)descr;
3653 sgen_hash_table_replace (&roots_hash [root_type], start, &new_root, NULL);
3656 SGEN_LOG (3, "Added root for range: %p-%p, descr: %p (%d/%d bytes)", start, new_root.end_root, descr, (int)size, (int)roots_size);
3663 mono_gc_register_root (char *start, size_t size, void *descr)
3665 return mono_gc_register_root_inner (start, size, descr, descr ? ROOT_TYPE_NORMAL : ROOT_TYPE_PINNED);
3669 mono_gc_register_root_wbarrier (char *start, size_t size, void *descr)
3671 return mono_gc_register_root_inner (start, size, descr, ROOT_TYPE_WBARRIER);
3675 mono_gc_deregister_root (char* addr)
3681 for (root_type = 0; root_type < ROOT_TYPE_NUM; ++root_type) {
3682 if (sgen_hash_table_remove (&roots_hash [root_type], addr, &root))
3683 roots_size -= (root.end_root - addr);
3689 * ######################################################################
3690 * ######## Thread handling (stop/start code)
3691 * ######################################################################
3694 unsigned int sgen_global_stop_count = 0;
3697 sgen_get_current_collection_generation (void)
3699 return current_collection_generation;
3703 mono_gc_set_gc_callbacks (MonoGCCallbacks *callbacks)
3705 gc_callbacks = *callbacks;
3709 mono_gc_get_gc_callbacks ()
3711 return &gc_callbacks;
3714 /* Variables holding start/end nursery so it won't have to be passed at every call */
3715 static void *scan_area_arg_start, *scan_area_arg_end;
3718 mono_gc_conservatively_scan_area (void *start, void *end)
3720 conservatively_pin_objects_from (start, end, scan_area_arg_start, scan_area_arg_end, PIN_TYPE_STACK);
3724 mono_gc_scan_object (void *obj, void *gc_data)
3726 UserCopyOrMarkData *data = gc_data;
3727 current_object_ops.copy_or_mark_object (&obj, data->queue);
3732 * Mark from thread stacks and registers.
3735 scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue)
3737 SgenThreadInfo *info;
3739 scan_area_arg_start = start_nursery;
3740 scan_area_arg_end = end_nursery;
3742 FOREACH_THREAD (info) {
3744 SGEN_LOG (3, "Skipping dead thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3747 if (info->gc_disabled) {
3748 SGEN_LOG (3, "GC disabled for thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start);
3751 if (mono_thread_info_run_state (info) != STATE_RUNNING) {
3752 SGEN_LOG (3, "Skipping non-running thread %p, range: %p-%p, size: %td (state %d)", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, mono_thread_info_run_state (info));
3755 SGEN_LOG (3, "Scanning thread %p, range: %p-%p, size: %td, pinned=%zd", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ());
3756 if (gc_callbacks.thread_mark_func && !conservative_stack_mark) {
3757 UserCopyOrMarkData data = { NULL, queue };
3758 gc_callbacks.thread_mark_func (info->runtime_data, info->stack_start, info->stack_end, precise, &data);
3759 } else if (!precise) {
3760 if (!conservative_stack_mark) {
3761 fprintf (stderr, "Precise stack mark not supported - disabling.\n");
3762 conservative_stack_mark = TRUE;
3764 conservatively_pin_objects_from (info->stack_start, info->stack_end, start_nursery, end_nursery, PIN_TYPE_STACK);
3769 conservatively_pin_objects_from ((void**)&info->ctx, (void**)&info->ctx + ARCH_NUM_REGS,
3770 start_nursery, end_nursery, PIN_TYPE_STACK);
3772 conservatively_pin_objects_from ((void**)&info->regs, (void**)&info->regs + ARCH_NUM_REGS,
3773 start_nursery, end_nursery, PIN_TYPE_STACK);
3776 } END_FOREACH_THREAD
3780 ptr_on_stack (void *ptr)
3782 gpointer stack_start = &stack_start;
3783 SgenThreadInfo *info = mono_thread_info_current ();
3785 if (ptr >= stack_start && ptr < (gpointer)info->stack_end)
3791 sgen_thread_register (SgenThreadInfo* info, void *addr)
3794 guint8 *staddr = NULL;
3796 #ifndef HAVE_KW_THREAD
3797 info->tlab_start = info->tlab_next = info->tlab_temp_end = info->tlab_real_end = NULL;
3799 g_assert (!mono_native_tls_get_value (thread_info_key));
3800 mono_native_tls_set_value (thread_info_key, info);
3802 sgen_thread_info = info;
3805 #ifdef SGEN_POSIX_STW
3806 info->stop_count = -1;
3810 info->stack_start = NULL;
3811 info->stopped_ip = NULL;
3812 info->stopped_domain = NULL;
3814 memset (&info->ctx, 0, sizeof (MonoContext));
3816 memset (&info->regs, 0, sizeof (info->regs));
3819 sgen_init_tlab_info (info);
3821 binary_protocol_thread_register ((gpointer)mono_thread_info_get_tid (info));
3823 /* On win32, stack_start_limit should be 0, since the stack can grow dynamically */
3824 mono_thread_info_get_stack_bounds (&staddr, &stsize);
3827 info->stack_start_limit = staddr;
3829 info->stack_end = staddr + stsize;
3831 gsize stack_bottom = (gsize)addr;
3832 stack_bottom += 4095;
3833 stack_bottom &= ~4095;
3834 info->stack_end = (char*)stack_bottom;
3837 #ifdef HAVE_KW_THREAD
3838 stack_end = info->stack_end;
3841 SGEN_LOG (3, "registered thread %p (%p) stack end %p", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end);
3843 if (gc_callbacks.thread_attach_func)
3844 info->runtime_data = gc_callbacks.thread_attach_func ();
3849 sgen_thread_detach (SgenThreadInfo *p)
3851 /* If a delegate is passed to native code and invoked on a thread we dont
3852 * know about, the jit will register it with mono_jit_thread_attach, but
3853 * we have no way of knowing when that thread goes away. SGen has a TSD
3854 * so we assume that if the domain is still registered, we can detach
3857 if (mono_domain_get ())
3858 mono_thread_detach_internal (mono_thread_internal_current ());
3862 sgen_thread_unregister (SgenThreadInfo *p)
3864 MonoNativeThreadId tid;
3866 tid = mono_thread_info_get_tid (p);
3867 binary_protocol_thread_unregister ((gpointer)tid);
3868 SGEN_LOG (3, "unregister thread %p (%p)", p, (gpointer)tid);
3870 #ifndef HAVE_KW_THREAD
3871 mono_native_tls_set_value (thread_info_key, NULL);
3873 sgen_thread_info = NULL;
3876 if (p->info.runtime_thread)
3877 mono_threads_add_joinable_thread ((gpointer)tid);
3879 if (gc_callbacks.thread_detach_func) {
3880 gc_callbacks.thread_detach_func (p->runtime_data);
3881 p->runtime_data = NULL;
3887 sgen_thread_attach (SgenThreadInfo *info)
3890 /*this is odd, can we get attached before the gc is inited?*/
3894 if (gc_callbacks.thread_attach_func && !info->runtime_data)
3895 info->runtime_data = gc_callbacks.thread_attach_func ();
3898 mono_gc_register_thread (void *baseptr)
3900 return mono_thread_info_attach (baseptr) != NULL;
3904 * mono_gc_set_stack_end:
3906 * Set the end of the current threads stack to STACK_END. The stack space between
3907 * STACK_END and the real end of the threads stack will not be scanned during collections.
3910 mono_gc_set_stack_end (void *stack_end)
3912 SgenThreadInfo *info;
3915 info = mono_thread_info_current ();
3917 g_assert (stack_end < info->stack_end);
3918 info->stack_end = stack_end;
3923 #if USE_PTHREAD_INTERCEPT
3927 mono_gc_pthread_create (pthread_t *new_thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
3929 return pthread_create (new_thread, attr, start_routine, arg);
3933 mono_gc_pthread_join (pthread_t thread, void **retval)
3935 return pthread_join (thread, retval);
3939 mono_gc_pthread_detach (pthread_t thread)
3941 return pthread_detach (thread);
3945 mono_gc_pthread_exit (void *retval)
3947 mono_thread_info_detach ();
3948 pthread_exit (retval);
3949 g_assert_not_reached ();
3952 #endif /* USE_PTHREAD_INTERCEPT */
3955 * ######################################################################
3956 * ######## Write barriers
3957 * ######################################################################
3961 * Note: the write barriers first do the needed GC work and then do the actual store:
3962 * this way the value is visible to the conservative GC scan after the write barrier
3963 * itself. If a GC interrupts the barrier in the middle, value will be kept alive by
3964 * the conservative scan, otherwise by the remembered set scan.
3967 mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* value)
3969 HEAVY_STAT (++stat_wbarrier_set_field);
3970 if (ptr_in_nursery (field_ptr)) {
3971 *(void**)field_ptr = value;
3974 SGEN_LOG (8, "Adding remset at %p", field_ptr);
3976 binary_protocol_wbarrier (field_ptr, value, value->vtable);
3978 remset.wbarrier_set_field (obj, field_ptr, value);
3982 mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* value)
3984 HEAVY_STAT (++stat_wbarrier_set_arrayref);
3985 if (ptr_in_nursery (slot_ptr)) {
3986 *(void**)slot_ptr = value;
3989 SGEN_LOG (8, "Adding remset at %p", slot_ptr);
3991 binary_protocol_wbarrier (slot_ptr, value, value->vtable);
3993 remset.wbarrier_set_arrayref (arr, slot_ptr, value);
3997 mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
3999 HEAVY_STAT (++stat_wbarrier_arrayref_copy);
4000 /*This check can be done without taking a lock since dest_ptr array is pinned*/
4001 if (ptr_in_nursery (dest_ptr) || count <= 0) {
4002 mono_gc_memmove_aligned (dest_ptr, src_ptr, count * sizeof (gpointer));
4006 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4007 if (binary_protocol_is_heavy_enabled ()) {
4009 for (i = 0; i < count; ++i) {
4010 gpointer dest = (gpointer*)dest_ptr + i;
4011 gpointer obj = *((gpointer*)src_ptr + i);
4013 binary_protocol_wbarrier (dest, obj, (gpointer)LOAD_VTABLE (obj));
4018 remset.wbarrier_arrayref_copy (dest_ptr, src_ptr, count);
4021 static char *found_obj;
4024 find_object_for_ptr_callback (char *obj, size_t size, void *user_data)
4026 char *ptr = user_data;
4028 if (ptr >= obj && ptr < obj + size) {
4029 g_assert (!found_obj);
4034 /* for use in the debugger */
4035 char* find_object_for_ptr (char *ptr);
4037 find_object_for_ptr (char *ptr)
4039 if (ptr >= nursery_section->data && ptr < nursery_section->end_data) {
4041 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data,
4042 find_object_for_ptr_callback, ptr, TRUE);
4048 sgen_los_iterate_objects (find_object_for_ptr_callback, ptr);
4053 * Very inefficient, but this is debugging code, supposed to
4054 * be called from gdb, so we don't care.
4057 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, find_object_for_ptr_callback, ptr);
4062 mono_gc_wbarrier_generic_nostore (gpointer ptr)
4066 HEAVY_STAT (++stat_wbarrier_generic_store);
4068 #ifdef XDOMAIN_CHECKS_IN_WBARRIER
4069 /* FIXME: ptr_in_heap must be called with the GC lock held */
4070 if (xdomain_checks && *(MonoObject**)ptr && ptr_in_heap (ptr)) {
4071 char *start = find_object_for_ptr (ptr);
4072 MonoObject *value = *(MonoObject**)ptr;
4076 MonoObject *obj = (MonoObject*)start;
4077 if (obj->vtable->domain != value->vtable->domain)
4078 g_assert (is_xdomain_ref_allowed (ptr, start, obj->vtable->domain));
4084 obj = *(gpointer*)ptr;
4086 binary_protocol_wbarrier (ptr, obj, (gpointer)LOAD_VTABLE (obj));
4088 if (ptr_in_nursery (ptr) || ptr_on_stack (ptr)) {
4089 SGEN_LOG (8, "Skipping remset at %p", ptr);
4094 * We need to record old->old pointer locations for the
4095 * concurrent collector.
4097 if (!ptr_in_nursery (obj) && !concurrent_collection_in_progress) {
4098 SGEN_LOG (8, "Skipping remset at %p", ptr);
4102 SGEN_LOG (8, "Adding remset at %p", ptr);
4104 remset.wbarrier_generic_nostore (ptr);
4108 mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value)
4110 SGEN_LOG (8, "Wbarrier store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4111 *(void**)ptr = value;
4112 if (ptr_in_nursery (value))
4113 mono_gc_wbarrier_generic_nostore (ptr);
4114 sgen_dummy_use (value);
4117 /* Same as mono_gc_wbarrier_generic_store () but performs the store
4118 * as an atomic operation with release semantics.
4121 mono_gc_wbarrier_generic_store_atomic (gpointer ptr, MonoObject *value)
4123 HEAVY_STAT (++stat_wbarrier_generic_store_atomic);
4125 SGEN_LOG (8, "Wbarrier atomic store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null");
4127 InterlockedWritePointer (ptr, value);
4129 if (ptr_in_nursery (value))
4130 mono_gc_wbarrier_generic_nostore (ptr);
4132 sgen_dummy_use (value);
4135 void mono_gc_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap)
4137 mword *dest = _dest;
4142 mono_gc_wbarrier_generic_store (dest, (MonoObject*)*src);
4147 size -= SIZEOF_VOID_P;
4152 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4154 #define HANDLE_PTR(ptr,obj) do { \
4155 gpointer o = *(gpointer*)(ptr); \
4157 gpointer d = ((char*)dest) + ((char*)(ptr) - (char*)(obj)); \
4158 binary_protocol_wbarrier (d, o, (gpointer) LOAD_VTABLE (o)); \
4163 scan_object_for_binary_protocol_copy_wbarrier (gpointer dest, char *start, mword desc)
4165 #define SCAN_OBJECT_NOVTABLE
4166 #include "sgen-scan-object.h"
4171 mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass *klass)
4173 HEAVY_STAT (++stat_wbarrier_value_copy);
4174 g_assert (klass->valuetype);
4176 SGEN_LOG (8, "Adding value remset at %p, count %d, descr %p for class %s (%p)", dest, count, klass->gc_descr, klass->name, klass);
4178 if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !SGEN_CLASS_HAS_REFERENCES (klass)) {
4179 size_t element_size = mono_class_value_size (klass, NULL);
4180 size_t size = count * element_size;
4181 mono_gc_memmove_atomic (dest, src, size);
4185 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4186 if (binary_protocol_is_heavy_enabled ()) {
4187 size_t element_size = mono_class_value_size (klass, NULL);
4189 for (i = 0; i < count; ++i) {
4190 scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size,
4191 (char*)src + i * element_size - sizeof (MonoObject),
4192 (mword) klass->gc_descr);
4197 remset.wbarrier_value_copy (dest, src, count, klass);
4201 * mono_gc_wbarrier_object_copy:
4203 * Write barrier to call when obj is the result of a clone or copy of an object.
4206 mono_gc_wbarrier_object_copy (MonoObject* obj, MonoObject *src)
4210 HEAVY_STAT (++stat_wbarrier_object_copy);
4212 if (ptr_in_nursery (obj) || ptr_on_stack (obj)) {
4213 size = mono_object_class (obj)->instance_size;
4214 mono_gc_memmove_aligned ((char*)obj + sizeof (MonoObject), (char*)src + sizeof (MonoObject),
4215 size - sizeof (MonoObject));
4219 #ifdef SGEN_HEAVY_BINARY_PROTOCOL
4220 if (binary_protocol_is_heavy_enabled ())
4221 scan_object_for_binary_protocol_copy_wbarrier (obj, (char*)src, (mword) src->vtable->gc_descr);
4224 remset.wbarrier_object_copy (obj, src);
4229 * ######################################################################
4230 * ######## Other mono public interface functions.
4231 * ######################################################################
4234 #define REFS_SIZE 128
4237 MonoGCReferences callback;
4241 MonoObject *refs [REFS_SIZE];
4242 uintptr_t offsets [REFS_SIZE];
4246 #define HANDLE_PTR(ptr,obj) do { \
4248 if (hwi->count == REFS_SIZE) { \
4249 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data); \
4253 hwi->offsets [hwi->count] = (char*)(ptr)-(char*)start; \
4254 hwi->refs [hwi->count++] = *(ptr); \
4259 collect_references (HeapWalkInfo *hwi, char *start, size_t size)
4261 mword desc = sgen_obj_get_descriptor (start);
4263 #include "sgen-scan-object.h"
4267 walk_references (char *start, size_t size, void *data)
4269 HeapWalkInfo *hwi = data;
4272 collect_references (hwi, start, size);
4273 if (hwi->count || !hwi->called)
4274 hwi->callback ((MonoObject*)start, mono_object_class (start), hwi->called? 0: size, hwi->count, hwi->refs, hwi->offsets, hwi->data);
4278 * mono_gc_walk_heap:
4279 * @flags: flags for future use
4280 * @callback: a function pointer called for each object in the heap
4281 * @data: a user data pointer that is passed to callback
4283 * This function can be used to iterate over all the live objects in the heap:
4284 * for each object, @callback is invoked, providing info about the object's
4285 * location in memory, its class, its size and the objects it references.
4286 * For each referenced object it's offset from the object address is
4287 * reported in the offsets array.
4288 * The object references may be buffered, so the callback may be invoked
4289 * multiple times for the same object: in all but the first call, the size
4290 * argument will be zero.
4291 * Note that this function can be only called in the #MONO_GC_EVENT_PRE_START_WORLD
4292 * profiler event handler.
4294 * Returns: a non-zero value if the GC doesn't support heap walking
4297 mono_gc_walk_heap (int flags, MonoGCReferences callback, void *data)
4302 hwi.callback = callback;
4305 sgen_clear_nursery_fragments ();
4306 sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, walk_references, &hwi, FALSE);
4308 major_collector.iterate_objects (ITERATE_OBJECTS_SWEEP_ALL, walk_references, &hwi);
4309 sgen_los_iterate_objects (walk_references, &hwi);
4315 mono_gc_collect (int generation)
4320 sgen_perform_collection (0, generation, "user request", TRUE);
4325 mono_gc_max_generation (void)
4331 mono_gc_collection_count (int generation)
4333 if (generation == 0)
4334 return gc_stats.minor_gc_count;
4335 return gc_stats.major_gc_count;
4339 mono_gc_get_used_size (void)
4343 tot = los_memory_usage;
4344 tot += nursery_section->next_data - nursery_section->data;
4345 tot += major_collector.get_used_size ();
4346 /* FIXME: account for pinned objects */
4352 mono_gc_get_los_limit (void)
4354 return MAX_SMALL_OBJ_SIZE;
4358 mono_gc_user_markers_supported (void)
4364 mono_object_is_alive (MonoObject* o)
4370 mono_gc_get_generation (MonoObject *obj)
4372 if (ptr_in_nursery (obj))
4378 mono_gc_enable_events (void)
4383 mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track)
4385 sgen_register_disappearing_link (obj, link_addr, track, FALSE);
4389 mono_gc_weak_link_remove (void **link_addr, gboolean track)
4391 sgen_register_disappearing_link (NULL, link_addr, track, FALSE);
4395 mono_gc_weak_link_get (void **link_addr)
4397 void * volatile *link_addr_volatile;
4401 link_addr_volatile = link_addr;
4402 ptr = (void*)*link_addr_volatile;
4404 * At this point we have a hidden pointer. If the GC runs
4405 * here, it will not recognize the hidden pointer as a
4406 * reference, and if the object behind it is not referenced
4407 * elsewhere, it will be freed. Once the world is restarted
4408 * we reveal the pointer, giving us a pointer to a freed
4409 * object. To make sure we don't return it, we load the
4410 * hidden pointer again. If it's still the same, we can be
4411 * sure the object reference is valid.
4414 obj = (MonoObject*) REVEAL_POINTER (ptr);
4418 mono_memory_barrier ();
4421 * During the second bridge processing step the world is
4422 * running again. That step processes all weak links once
4423 * more to null those that refer to dead objects. Before that
4424 * is completed, those links must not be followed, so we
4425 * conservatively wait for bridge processing when any weak
4426 * link is dereferenced.
4428 if (G_UNLIKELY (bridge_processing_in_progress))
4429 mono_gc_wait_for_bridge_processing ();
4431 if ((void*)*link_addr_volatile != ptr)
4438 mono_gc_ephemeron_array_add (MonoObject *obj)
4440 EphemeronLinkNode *node;
4444 node = sgen_alloc_internal (INTERNAL_MEM_EPHEMERON_LINK);
4449 node->array = (char*)obj;
4450 node->next = ephemeron_list;
4451 ephemeron_list = node;
4453 SGEN_LOG (5, "Registered ephemeron array %p", obj);
4460 mono_gc_set_allow_synchronous_major (gboolean flag)
4462 if (!major_collector.is_concurrent)
4465 allow_synchronous_major = flag;
4470 mono_gc_invoke_with_gc_lock (MonoGCLockedCallbackFunc func, void *data)
4474 result = func (data);
4475 UNLOCK_INTERRUPTION;
4480 mono_gc_is_gc_thread (void)
4484 result = mono_thread_info_current () != NULL;
4490 is_critical_method (MonoMethod *method)
4492 return mono_runtime_is_critical_method (method) || sgen_is_critical_method (method);
4496 sgen_env_var_error (const char *env_var, const char *fallback, const char *description_format, ...)
4500 va_start (ap, description_format);
4502 fprintf (stderr, "Warning: In environment variable `%s': ", env_var);
4503 vfprintf (stderr, description_format, ap);
4505 fprintf (stderr, " - %s", fallback);
4506 fprintf (stderr, "\n");
4512 parse_double_in_interval (const char *env_var, const char *opt_name, const char *opt, double min, double max, double *result)
4515 double val = strtod (opt, &endptr);
4516 if (endptr == opt) {
4517 sgen_env_var_error (env_var, "Using default value.", "`%s` must be a number.", opt_name);
4520 else if (val < min || val > max) {
4521 sgen_env_var_error (env_var, "Using default value.", "`%s` must be between %.2f - %.2f.", opt_name, min, max);
4529 mono_gc_base_init (void)
4531 MonoThreadInfoCallbacks cb;
4534 char *major_collector_opt = NULL;
4535 char *minor_collector_opt = NULL;
4536 size_t max_heap = 0;
4537 size_t soft_limit = 0;
4540 gboolean debug_print_allowance = FALSE;
4541 double allowance_ratio = 0, save_target = 0;
4542 gboolean have_split_nursery = FALSE;
4543 gboolean cement_enabled = TRUE;
4546 result = InterlockedCompareExchange (&gc_initialized, -1, 0);
4549 /* already inited */
4552 /* being inited by another thread */
4556 /* we will init it */
4559 g_assert_not_reached ();
4561 } while (result != 0);
4563 SGEN_TV_GETTIME (sgen_init_timestamp);
4565 LOCK_INIT (gc_mutex);
4567 pagesize = mono_pagesize ();
4568 gc_debug_file = stderr;
4570 cb.thread_register = sgen_thread_register;
4571 cb.thread_detach = sgen_thread_detach;
4572 cb.thread_unregister = sgen_thread_unregister;
4573 cb.thread_attach = sgen_thread_attach;
4574 cb.mono_method_is_critical = (gpointer)is_critical_method;
4576 cb.thread_exit = mono_gc_pthread_exit;
4577 cb.mono_gc_pthread_create = (gpointer)mono_gc_pthread_create;
4580 mono_threads_init (&cb, sizeof (SgenThreadInfo));
4582 LOCK_INIT (sgen_interruption_mutex);
4583 LOCK_INIT (pin_queue_mutex);
4585 if ((env = g_getenv (MONO_GC_PARAMS_NAME))) {
4586 opts = g_strsplit (env, ",", -1);
4587 for (ptr = opts; *ptr; ++ptr) {
4589 if (g_str_has_prefix (opt, "major=")) {
4590 opt = strchr (opt, '=') + 1;
4591 major_collector_opt = g_strdup (opt);
4592 } else if (g_str_has_prefix (opt, "minor=")) {
4593 opt = strchr (opt, '=') + 1;
4594 minor_collector_opt = g_strdup (opt);
4602 sgen_init_internal_allocator ();
4603 sgen_init_nursery_allocator ();
4604 sgen_init_fin_weak_hash ();
4606 sgen_init_hash_table ();
4608 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_SECTION, SGEN_SIZEOF_GC_MEM_SECTION);
4609 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_FINALIZE_READY_ENTRY, sizeof (FinalizeReadyEntry));
4610 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_GRAY_QUEUE, sizeof (GrayQueueSection));
4611 sgen_register_fixed_internal_mem_type (INTERNAL_MEM_EPHEMERON_LINK, sizeof (EphemeronLinkNode));
4613 #ifndef HAVE_KW_THREAD
4614 mono_native_tls_alloc (&thread_info_key, NULL);
4615 #if defined(__APPLE__) || defined (HOST_WIN32)
4617 * CEE_MONO_TLS requires the tls offset, not the key, so the code below only works on darwin,
4618 * where the two are the same.
4620 mono_tls_key_set_offset (TLS_KEY_SGEN_THREAD_INFO, thread_info_key);
4624 int tls_offset = -1;
4625 MONO_THREAD_VAR_OFFSET (sgen_thread_info, tls_offset);
4626 mono_tls_key_set_offset (TLS_KEY_SGEN_THREAD_INFO, tls_offset);
4631 * This needs to happen before any internal allocations because
4632 * it inits the small id which is required for hazard pointer
4637 mono_thread_info_attach (&dummy);
4639 if (!minor_collector_opt) {
4640 sgen_simple_nursery_init (&sgen_minor_collector);
4642 if (!strcmp (minor_collector_opt, "simple")) {
4644 sgen_simple_nursery_init (&sgen_minor_collector);
4645 } else if (!strcmp (minor_collector_opt, "split")) {
4646 sgen_split_nursery_init (&sgen_minor_collector);
4647 have_split_nursery = TRUE;
4649 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `simple` instead.", "Unknown minor collector `%s'.", minor_collector_opt);
4650 goto use_simple_nursery;
4654 if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep")) {
4655 use_marksweep_major:
4656 sgen_marksweep_init (&major_collector);
4657 } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-conc")) {
4658 sgen_marksweep_conc_init (&major_collector);
4660 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using `marksweep` instead.", "Unknown major collector `%s'.", major_collector_opt);
4661 goto use_marksweep_major;
4664 ///* Keep this the default for now */
4665 /* Precise marking is broken on all supported targets. Disable until fixed. */
4666 conservative_stack_mark = TRUE;
4668 sgen_nursery_size = DEFAULT_NURSERY_SIZE;
4671 gboolean usage_printed = FALSE;
4673 for (ptr = opts; *ptr; ++ptr) {
4675 if (!strcmp (opt, ""))
4677 if (g_str_has_prefix (opt, "major="))
4679 if (g_str_has_prefix (opt, "minor="))
4681 if (g_str_has_prefix (opt, "max-heap-size=")) {
4682 size_t max_heap_candidate = 0;
4683 opt = strchr (opt, '=') + 1;
4684 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &max_heap_candidate)) {
4685 max_heap = (max_heap_candidate + mono_pagesize () - 1) & ~(size_t)(mono_pagesize () - 1);
4686 if (max_heap != max_heap_candidate)
4687 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Rounding up.", "`max-heap-size` size must be a multiple of %d.", mono_pagesize ());
4689 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`max-heap-size` must be an integer.");
4693 if (g_str_has_prefix (opt, "soft-heap-limit=")) {
4694 opt = strchr (opt, '=') + 1;
4695 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &soft_limit)) {
4696 if (soft_limit <= 0) {
4697 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be positive.");
4701 sgen_env_var_error (MONO_GC_PARAMS_NAME, NULL, "`soft-heap-limit` must be an integer.");
4705 if (g_str_has_prefix (opt, "stack-mark=")) {
4706 opt = strchr (opt, '=') + 1;
4707 if (!strcmp (opt, "precise")) {
4708 conservative_stack_mark = FALSE;
4709 } else if (!strcmp (opt, "conservative")) {
4710 conservative_stack_mark = TRUE;
4712 sgen_env_var_error (MONO_GC_PARAMS_NAME, conservative_stack_mark ? "Using `conservative`." : "Using `precise`.",
4713 "Invalid value `%s` for `stack-mark` option, possible values are: `precise`, `conservative`.", opt);
4717 if (g_str_has_prefix (opt, "bridge-implementation=")) {
4718 opt = strchr (opt, '=') + 1;
4719 sgen_set_bridge_implementation (opt);
4722 if (g_str_has_prefix (opt, "toggleref-test")) {
4723 sgen_register_test_toggleref_callback ();
4728 if (g_str_has_prefix (opt, "nursery-size=")) {
4730 opt = strchr (opt, '=') + 1;
4731 if (*opt && mono_gc_parse_environment_string_extract_number (opt, &val)) {
4732 #ifdef SGEN_ALIGN_NURSERY
4733 if ((val & (val - 1))) {
4734 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be a power of two.");
4738 if (val < SGEN_MAX_NURSERY_WASTE) {
4739 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.",
4740 "`nursery-size` must be at least %d bytes.", SGEN_MAX_NURSERY_WASTE);
4744 sgen_nursery_size = val;
4745 sgen_nursery_bits = 0;
4746 while (ONE_P << (++ sgen_nursery_bits) != sgen_nursery_size)
4749 sgen_nursery_size = val;
4752 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`nursery-size` must be an integer.");
4758 if (g_str_has_prefix (opt, "save-target-ratio=")) {
4760 opt = strchr (opt, '=') + 1;
4761 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "save-target-ratio", opt,
4762 SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO, &val)) {
4767 if (g_str_has_prefix (opt, "default-allowance-ratio=")) {
4769 opt = strchr (opt, '=') + 1;
4770 if (parse_double_in_interval (MONO_GC_PARAMS_NAME, "default-allowance-ratio", opt,
4771 SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, &val)) {
4772 allowance_ratio = val;
4776 if (g_str_has_prefix (opt, "allow-synchronous-major=")) {
4777 if (!major_collector.is_concurrent) {
4778 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "`allow-synchronous-major` is only valid for the concurrent major collector.");
4782 opt = strchr (opt, '=') + 1;
4784 if (!strcmp (opt, "yes")) {
4785 allow_synchronous_major = TRUE;
4786 } else if (!strcmp (opt, "no")) {
4787 allow_synchronous_major = FALSE;
4789 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Using default value.", "`allow-synchronous-major` must be either `yes' or `no'.");
4794 if (!strcmp (opt, "cementing")) {
4795 cement_enabled = TRUE;
4798 if (!strcmp (opt, "no-cementing")) {
4799 cement_enabled = FALSE;
4803 if (major_collector.handle_gc_param && major_collector.handle_gc_param (opt))
4806 if (sgen_minor_collector.handle_gc_param && sgen_minor_collector.handle_gc_param (opt))
4809 sgen_env_var_error (MONO_GC_PARAMS_NAME, "Ignoring.", "Unknown option `%s`.", opt);
4814 fprintf (stderr, "\n%s must be a comma-delimited list of one or more of the following:\n", MONO_GC_PARAMS_NAME);
4815 fprintf (stderr, " max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
4816 fprintf (stderr, " soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n");
4817 fprintf (stderr, " nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
4818 fprintf (stderr, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-conc', `marksweep-par')\n");
4819 fprintf (stderr, " minor=COLLECTOR (where COLLECTOR is `simple' or `split')\n");
4820 fprintf (stderr, " wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n");
4821 fprintf (stderr, " stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n");
4822 fprintf (stderr, " [no-]cementing\n");
4823 if (major_collector.is_concurrent)
4824 fprintf (stderr, " allow-synchronous-major=FLAG (where FLAG is `yes' or `no')\n");
4825 if (major_collector.print_gc_param_usage)
4826 major_collector.print_gc_param_usage ();
4827 if (sgen_minor_collector.print_gc_param_usage)
4828 sgen_minor_collector.print_gc_param_usage ();
4829 fprintf (stderr, " Experimental options:\n");
4830 fprintf (stderr, " save-target-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_SAVE_TARGET_RATIO, SGEN_MAX_SAVE_TARGET_RATIO);
4831 fprintf (stderr, " default-allowance-ratio=R (where R must be between %.2f - %.2f).\n", SGEN_MIN_ALLOWANCE_NURSERY_SIZE_RATIO, SGEN_MAX_ALLOWANCE_NURSERY_SIZE_RATIO);
4832 fprintf (stderr, "\n");
4834 usage_printed = TRUE;
4839 if (major_collector.is_concurrent)
4840 sgen_workers_init (1);
4842 if (major_collector_opt)
4843 g_free (major_collector_opt);
4845 if (minor_collector_opt)
4846 g_free (minor_collector_opt);
4850 sgen_cement_init (cement_enabled);
4852 if ((env = g_getenv (MONO_GC_DEBUG_NAME))) {
4853 gboolean usage_printed = FALSE;
4855 opts = g_strsplit (env, ",", -1);
4856 for (ptr = opts; ptr && *ptr; ptr ++) {
4858 if (!strcmp (opt, ""))
4860 if (opt [0] >= '0' && opt [0] <= '9') {
4861 gc_debug_level = atoi (opt);
4867 char *rf = g_strdup_printf ("%s.%d", opt, GetCurrentProcessId ());
4869 char *rf = g_strdup_printf ("%s.%d", opt, getpid ());
4871 gc_debug_file = fopen (rf, "wb");
4873 gc_debug_file = stderr;
4876 } else if (!strcmp (opt, "print-allowance")) {
4877 debug_print_allowance = TRUE;
4878 } else if (!strcmp (opt, "print-pinning")) {
4879 do_pin_stats = TRUE;
4880 } else if (!strcmp (opt, "verify-before-allocs")) {
4881 verify_before_allocs = 1;
4882 has_per_allocation_action = TRUE;
4883 } else if (g_str_has_prefix (opt, "verify-before-allocs=")) {
4884 char *arg = strchr (opt, '=') + 1;
4885 verify_before_allocs = atoi (arg);
4886 has_per_allocation_action = TRUE;
4887 } else if (!strcmp (opt, "collect-before-allocs")) {
4888 collect_before_allocs = 1;
4889 has_per_allocation_action = TRUE;
4890 } else if (g_str_has_prefix (opt, "collect-before-allocs=")) {
4891 char *arg = strchr (opt, '=') + 1;
4892 has_per_allocation_action = TRUE;
4893 collect_before_allocs = atoi (arg);
4894 } else if (!strcmp (opt, "verify-before-collections")) {
4895 whole_heap_check_before_collection = TRUE;
4896 } else if (!strcmp (opt, "check-at-minor-collections")) {
4897 consistency_check_at_minor_collection = TRUE;
4898 nursery_clear_policy = CLEAR_AT_GC;
4899 } else if (!strcmp (opt, "mod-union-consistency-check")) {
4900 if (!major_collector.is_concurrent) {
4901 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`mod-union-consistency-check` only works with concurrent major collector.");
4904 mod_union_consistency_check = TRUE;
4905 } else if (!strcmp (opt, "check-mark-bits")) {
4906 check_mark_bits_after_major_collection = TRUE;
4907 } else if (!strcmp (opt, "check-nursery-pinned")) {
4908 check_nursery_objects_pinned = TRUE;
4909 } else if (!strcmp (opt, "xdomain-checks")) {
4910 xdomain_checks = TRUE;
4911 } else if (!strcmp (opt, "clear-at-gc")) {
4912 nursery_clear_policy = CLEAR_AT_GC;
4913 } else if (!strcmp (opt, "clear-nursery-at-gc")) {
4914 nursery_clear_policy = CLEAR_AT_GC;
4915 } else if (!strcmp (opt, "clear-at-tlab-creation")) {
4916 nursery_clear_policy = CLEAR_AT_TLAB_CREATION;
4917 } else if (!strcmp (opt, "debug-clear-at-tlab-creation")) {
4918 nursery_clear_policy = CLEAR_AT_TLAB_CREATION_DEBUG;
4919 } else if (!strcmp (opt, "check-scan-starts")) {
4920 do_scan_starts_check = TRUE;
4921 } else if (!strcmp (opt, "verify-nursery-at-minor-gc")) {
4922 do_verify_nursery = TRUE;
4923 } else if (!strcmp (opt, "check-concurrent")) {
4924 if (!major_collector.is_concurrent) {
4925 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "`check-concurrent` only works with concurrent major collectors.");
4928 do_concurrent_checks = TRUE;
4929 } else if (!strcmp (opt, "dump-nursery-at-minor-gc")) {
4930 do_dump_nursery_content = TRUE;
4931 } else if (!strcmp (opt, "no-managed-allocator")) {
4932 sgen_set_use_managed_allocator (FALSE);
4933 } else if (!strcmp (opt, "disable-minor")) {
4934 disable_minor_collections = TRUE;
4935 } else if (!strcmp (opt, "disable-major")) {
4936 disable_major_collections = TRUE;
4937 } else if (g_str_has_prefix (opt, "heap-dump=")) {
4938 char *filename = strchr (opt, '=') + 1;
4939 nursery_clear_policy = CLEAR_AT_GC;
4940 heap_dump_file = fopen (filename, "w");
4941 if (heap_dump_file) {
4942 fprintf (heap_dump_file, "<sgen-dump>\n");
4943 do_pin_stats = TRUE;
4945 } else if (g_str_has_prefix (opt, "binary-protocol=")) {
4946 char *filename = strchr (opt, '=') + 1;
4947 char *colon = strrchr (filename, ':');
4950 if (!mono_gc_parse_environment_string_extract_number (colon + 1, &limit)) {
4951 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring limit.", "Binary protocol file size limit must be an integer.");
4956 binary_protocol_init (filename, (long long)limit);
4957 } else if (!sgen_bridge_handle_gc_debug (opt)) {
4958 sgen_env_var_error (MONO_GC_DEBUG_NAME, "Ignoring.", "Unknown option `%s`.", opt);
4963 fprintf (stderr, "\n%s must be of the format [<l>[:<filename>]|<option>]+ where <l> is a debug level 0-9.\n", MONO_GC_DEBUG_NAME);
4964 fprintf (stderr, "Valid <option>s are:\n");
4965 fprintf (stderr, " collect-before-allocs[=<n>]\n");
4966 fprintf (stderr, " verify-before-allocs[=<n>]\n");
4967 fprintf (stderr, " check-at-minor-collections\n");
4968 fprintf (stderr, " check-mark-bits\n");
4969 fprintf (stderr, " check-nursery-pinned\n");
4970 fprintf (stderr, " verify-before-collections\n");
4971 fprintf (stderr, " verify-nursery-at-minor-gc\n");
4972 fprintf (stderr, " dump-nursery-at-minor-gc\n");
4973 fprintf (stderr, " disable-minor\n");
4974 fprintf (stderr, " disable-major\n");
4975 fprintf (stderr, " xdomain-checks\n");
4976 fprintf (stderr, " check-concurrent\n");
4977 fprintf (stderr, " clear-[nursery-]at-gc\n");
4978 fprintf (stderr, " clear-at-tlab-creation\n");
4979 fprintf (stderr, " debug-clear-at-tlab-creation\n");
4980 fprintf (stderr, " check-scan-starts\n");
4981 fprintf (stderr, " no-managed-allocator\n");
4982 fprintf (stderr, " print-allowance\n");
4983 fprintf (stderr, " print-pinning\n");
4984 fprintf (stderr, " heap-dump=<filename>\n");
4985 fprintf (stderr, " binary-protocol=<filename>[:<file-size-limit>]\n");
4986 sgen_bridge_print_gc_debug_usage ();
4987 fprintf (stderr, "\n");
4989 usage_printed = TRUE;
4995 if (major_collector.post_param_init)
4996 major_collector.post_param_init (&major_collector);
4998 sgen_memgov_init (max_heap, soft_limit, debug_print_allowance, allowance_ratio, save_target);
5000 memset (&remset, 0, sizeof (remset));
5002 sgen_card_table_init (&remset);
5008 mono_gc_get_gc_name (void)
5013 static MonoMethod *write_barrier_method;
5016 sgen_is_critical_method (MonoMethod *method)
5018 return (method == write_barrier_method || sgen_is_managed_allocator (method));
5022 sgen_has_critical_method (void)
5024 return write_barrier_method || sgen_has_managed_allocator ();
5030 emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels)
5032 memset (nursery_check_return_labels, 0, sizeof (int) * 3);
5033 #ifdef SGEN_ALIGN_NURSERY
5034 // if (ptr_in_nursery (ptr)) return;
5036 * Masking out the bits might be faster, but we would have to use 64 bit
5037 * immediates, which might be slower.
5039 mono_mb_emit_ldarg (mb, 0);
5040 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5041 mono_mb_emit_byte (mb, CEE_SHR_UN);
5042 mono_mb_emit_ptr (mb, (gpointer)((mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS));
5043 nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ);
5045 if (!major_collector.is_concurrent) {
5046 // if (!ptr_in_nursery (*ptr)) return;
5047 mono_mb_emit_ldarg (mb, 0);
5048 mono_mb_emit_byte (mb, CEE_LDIND_I);
5049 mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS);
5050 mono_mb_emit_byte (mb, CEE_SHR_UN);
5051 mono_mb_emit_ptr (mb, (gpointer)((mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS));
5052 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN);
5055 int label_continue1, label_continue2;
5056 int dereferenced_var;
5058 // if (ptr < (sgen_get_nursery_start ())) goto continue;
5059 mono_mb_emit_ldarg (mb, 0);
5060 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5061 label_continue_1 = mono_mb_emit_branch (mb, CEE_BLT);
5063 // if (ptr >= sgen_get_nursery_end ())) goto continue;
5064 mono_mb_emit_ldarg (mb, 0);
5065 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5066 label_continue_2 = mono_mb_emit_branch (mb, CEE_BGE);
5069 nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BR);
5072 mono_mb_patch_branch (mb, label_continue_1);
5073 mono_mb_patch_branch (mb, label_continue_2);
5075 // Dereference and store in local var
5076 dereferenced_var = mono_mb_add_local (mb, &mono_defaults.int_class->byval_arg);
5077 mono_mb_emit_ldarg (mb, 0);
5078 mono_mb_emit_byte (mb, CEE_LDIND_I);
5079 mono_mb_emit_stloc (mb, dereferenced_var);
5081 if (!major_collector.is_concurrent) {
5082 // if (*ptr < sgen_get_nursery_start ()) return;
5083 mono_mb_emit_ldloc (mb, dereferenced_var);
5084 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ());
5085 nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT);
5087 // if (*ptr >= sgen_get_nursery_end ()) return;
5088 mono_mb_emit_ldloc (mb, dereferenced_var);
5089 mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ());
5090 nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE);
5097 mono_gc_get_write_barrier (void)
5100 MonoMethodBuilder *mb;
5101 MonoMethodSignature *sig;
5102 #ifdef MANAGED_WBARRIER
5103 int i, nursery_check_labels [3];
5105 #ifdef HAVE_KW_THREAD
5106 int stack_end_offset = -1;
5108 MONO_THREAD_VAR_OFFSET (stack_end, stack_end_offset);
5109 g_assert (stack_end_offset != -1);
5113 // FIXME: Maybe create a separate version for ctors (the branch would be
5114 // correctly predicted more times)
5115 if (write_barrier_method)
5116 return write_barrier_method;
5118 /* Create the IL version of mono_gc_barrier_generic_store () */
5119 sig = mono_metadata_signature_alloc (mono_defaults.corlib, 1);
5120 sig->ret = &mono_defaults.void_class->byval_arg;
5121 sig->params [0] = &mono_defaults.int_class->byval_arg;
5123 mb = mono_mb_new (mono_defaults.object_class, "wbarrier", MONO_WRAPPER_WRITE_BARRIER);
5126 #ifdef MANAGED_WBARRIER
5127 emit_nursery_check (mb, nursery_check_labels);
5129 addr = sgen_cardtable + ((address >> CARD_BITS) & CARD_MASK)
5133 LDC_PTR sgen_cardtable
5135 address >> CARD_BITS
5139 if (SGEN_HAVE_OVERLAPPING_CARDS) {
5140 LDC_PTR card_table_mask
5147 mono_mb_emit_ptr (mb, sgen_cardtable);
5148 mono_mb_emit_ldarg (mb, 0);
5149 mono_mb_emit_icon (mb, CARD_BITS);
5150 mono_mb_emit_byte (mb, CEE_SHR_UN);
5151 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
5152 mono_mb_emit_ptr (mb, (gpointer)CARD_MASK);
5153 mono_mb_emit_byte (mb, CEE_AND);
5155 mono_mb_emit_byte (mb, CEE_ADD);
5156 mono_mb_emit_icon (mb, 1);
5157 mono_mb_emit_byte (mb, CEE_STIND_I1);
5160 for (i = 0; i < 3; ++i) {
5161 if (nursery_check_labels [i])
5162 mono_mb_patch_branch (mb, nursery_check_labels [i]);
5164 mono_mb_emit_byte (mb, CEE_RET);
5166 mono_mb_emit_ldarg (mb, 0);
5167 mono_mb_emit_icall (mb, mono_gc_wbarrier_generic_nostore);
5168 mono_mb_emit_byte (mb, CEE_RET);
5171 res = mono_mb_create_method (mb, sig, 16);
5175 if (write_barrier_method) {
5176 /* Already created */
5177 mono_free_method (res);
5179 /* double-checked locking */
5180 mono_memory_barrier ();
5181 write_barrier_method = res;
5185 return write_barrier_method;
5189 mono_gc_get_description (void)
5191 return g_strdup ("sgen");
5195 mono_gc_set_desktop_mode (void)
5200 mono_gc_is_moving (void)
5206 mono_gc_is_disabled (void)
5212 BOOL APIENTRY mono_gc_dllmain (HMODULE module_handle, DWORD reason, LPVOID reserved)
5219 sgen_get_nursery_clear_policy (void)
5221 return nursery_clear_policy;
5225 sgen_get_array_fill_vtable (void)
5227 if (!array_fill_vtable) {
5228 static MonoClass klass;
5229 static MonoVTable vtable;
5232 MonoDomain *domain = mono_get_root_domain ();
5235 klass.element_class = mono_defaults.byte_class;
5237 klass.instance_size = sizeof (MonoArray);
5238 klass.sizes.element_size = 1;
5239 klass.name = "array_filler_type";
5241 vtable.klass = &klass;
5243 vtable.gc_descr = mono_gc_make_descr_for_array (TRUE, &bmap, 0, 1);
5246 array_fill_vtable = &vtable;
5248 return array_fill_vtable;
5258 sgen_gc_unlock (void)
5260 gboolean try_free = sgen_try_free_some_memory;
5261 sgen_try_free_some_memory = FALSE;
5262 mono_mutex_unlock (&gc_mutex);
5263 MONO_GC_UNLOCKED ();
5265 mono_thread_hazardous_try_free_some ();
5269 sgen_major_collector_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
5271 major_collector.iterate_live_block_ranges (callback);
5275 sgen_major_collector_scan_card_table (SgenGrayQueue *queue)
5277 major_collector.scan_card_table (FALSE, queue);
5281 sgen_get_major_collector (void)
5283 return &major_collector;
5286 void mono_gc_set_skip_thread (gboolean skip)
5288 SgenThreadInfo *info = mono_thread_info_current ();
5291 info->gc_disabled = skip;
5296 sgen_get_remset (void)
5302 mono_gc_get_vtable_bits (MonoClass *class)
5305 /* FIXME move this to the bridge code */
5306 if (sgen_need_bridge_processing ()) {
5307 switch (sgen_bridge_class_kind (class)) {
5308 case GC_BRIDGE_TRANSPARENT_BRIDGE_CLASS:
5309 case GC_BRIDGE_OPAQUE_BRIDGE_CLASS:
5310 res = SGEN_GC_BIT_BRIDGE_OBJECT;
5312 case GC_BRIDGE_OPAQUE_CLASS:
5313 res = SGEN_GC_BIT_BRIDGE_OPAQUE_OBJECT;
5317 if (fin_callbacks.is_class_finalization_aware) {
5318 if (fin_callbacks.is_class_finalization_aware (class))
5319 res |= SGEN_GC_BIT_FINALIZER_AWARE;
5325 mono_gc_register_altstack (gpointer stack, gint32 stack_size, gpointer altstack, gint32 altstack_size)
5332 sgen_check_whole_heap_stw (void)
5334 sgen_stop_world (0);
5335 sgen_clear_nursery_fragments ();
5336 sgen_check_whole_heap (FALSE);
5337 sgen_restart_world (0, NULL);
5341 sgen_gc_event_moves (void)
5343 if (moved_objects_idx) {
5344 mono_profiler_gc_moves (moved_objects, moved_objects_idx);
5345 moved_objects_idx = 0;
5350 sgen_timestamp (void)
5352 SGEN_TV_DECLARE (timestamp);
5353 SGEN_TV_GETTIME (timestamp);
5354 return SGEN_TV_ELAPSED (sgen_init_timestamp, timestamp);
5358 mono_gc_register_finalizer_callbacks (MonoGCFinalizerCallbacks *callbacks)
5360 if (callbacks->version != MONO_GC_FINALIZER_EXTENSION_VERSION)
5361 g_error ("Invalid finalizer callback version. Expected %d but got %d\n", MONO_GC_FINALIZER_EXTENSION_VERSION, callbacks->version);
5363 fin_callbacks = *callbacks;
5366 #endif /* HAVE_SGEN_GC */