X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mono%2Fmetadata%2Fsgen-gc.c;h=32416bd40863da850309ce214f5c0d92f684155b;hb=74ce6bc4e97251f9498efcec5bc5219a76c9d8b3;hp=0fa833603e50607ae4797daeb679232a38dc31af;hpb=936d70b46274f033c3dec6234d130843677ab8f9;p=mono.git diff --git a/mono/metadata/sgen-gc.c b/mono/metadata/sgen-gc.c index 0fa833603e5..32416bd4086 100644 --- a/mono/metadata/sgen-gc.c +++ b/mono/metadata/sgen-gc.c @@ -13,40 +13,23 @@ * Copyright (c) 1996 by Silicon Graphics. All rights reserved. * Copyright (c) 1998 by Fergus Henderson. All rights reserved. * Copyright (c) 2000-2004 by Hewlett-Packard Company. All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - * - * * Copyright 2001-2003 Ximian, Inc * Copyright 2003-2010 Novell, Inc. * Copyright 2011 Xamarin, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * Copyright (C) 2012 Xamarin Inc * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License 2.0 as published by the Free Software Foundation; + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License 2.0 along with this library; if not, write to the Free + * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. * * Important: allocation provides always zeroed memory, having to do * a memset after allocation is deadly for performance. @@ -217,6 +200,7 @@ #include "metadata/sgen-archdep.h" #include "metadata/sgen-bridge.h" #include "metadata/sgen-memory-governor.h" +#include "metadata/sgen-hash-table.h" #include "metadata/mono-gc.h" #include "metadata/method-builder.h" #include "metadata/profiler-private.h" @@ -368,16 +352,6 @@ mono_gc_flush_info (void) NurseryClearPolicy nursery_clear_policy = CLEAR_AT_TLAB_CREATION; -/* the runtime can register areas of memory as roots: we keep two lists of roots, - * a pinned root set for conservatively scanned roots and a normal one for - * precisely scanned roots (currently implemented as a single list). - */ -typedef struct _RootRecord RootRecord; -struct _RootRecord { - char *end_root; - mword root_desc; -}; - #define object_is_forwarded SGEN_OBJECT_IS_FORWARDED #define object_is_pinned SGEN_OBJECT_IS_PINNED #define pin_object SGEN_PIN_OBJECT @@ -423,7 +397,7 @@ GCMemSection *nursery_section = NULL; static mword lowest_heap_address = ~(mword)0; static mword highest_heap_address = 0; -static LOCK_DECLARE (interruption_mutex); +LOCK_DECLARE (sgen_interruption_mutex); static LOCK_DECLARE (pin_queue_mutex); #define LOCK_PIN_QUEUE mono_mutex_lock (&pin_queue_mutex) @@ -448,14 +422,7 @@ typedef struct { } Ephemeron; int current_collection_generation = -1; - -/* - * The link pointer is hidden by negating each bit. We use the lowest - * bit of the link (before negation) to store whether it needs - * resurrection tracking. - */ -#define HIDE_POINTER(p,t) ((gpointer)(~((gulong)(p)|((t)?1:0)))) -#define REVEAL_POINTER(p) ((gpointer)((~(gulong)(p))&~3L)) +volatile gboolean concurrent_collection_in_progress = FALSE; /* objects that are ready to be finalized */ static FinalizeReadyEntry *fin_ready_list = NULL; @@ -463,21 +430,11 @@ static FinalizeReadyEntry *critical_fin_list = NULL; static EphemeronLinkNode *ephemeron_list; -static int num_ready_finalizers = 0; -static int no_finalize = 0; - -enum { - ROOT_TYPE_NORMAL = 0, /* "normal" roots */ - ROOT_TYPE_PINNED = 1, /* roots without a GC descriptor */ - ROOT_TYPE_WBARRIER = 2, /* roots with a write barrier */ - ROOT_TYPE_NUM -}; - /* registered roots: the key to the hash is the root start address */ /* * Different kinds of roots are kept separate to speed up pin_from_roots () for example. */ -static SgenHashTable roots_hash [ROOT_TYPE_NUM] = { +SgenHashTable roots_hash [ROOT_TYPE_NUM] = { SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL), SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL), SGEN_HASH_TABLE_INIT (INTERNAL_MEM_ROOTS_TABLE, INTERNAL_MEM_ROOT_RECORD, sizeof (RootRecord), mono_aligned_addr_hash, NULL) @@ -486,7 +443,7 @@ static mword roots_size = 0; /* amount of memory in the root set */ #define GC_ROOT_NUM 32 typedef struct { - int count; + int count; /* must be the first field */ void *objects [GC_ROOT_NUM]; int root_types [GC_ROOT_NUM]; uintptr_t extra_info [GC_ROOT_NUM]; @@ -514,7 +471,7 @@ add_profile_gc_root (GCRootReport *report, void *object, int rtype, uintptr_t ex MonoNativeTlsKey thread_info_key; #ifdef HAVE_KW_THREAD -__thread SgenThreadInfo *thread_info; +__thread SgenThreadInfo *sgen_thread_info; __thread gpointer *store_remset_buffer; __thread long store_remset_buffer_index; __thread char *stack_end; @@ -571,30 +528,16 @@ align_pointer (void *ptr) typedef SgenGrayQueue GrayQueue; /* forward declarations */ -static int stop_world (int generation); -static int restart_world (int generation, GGTimingInfo *timing); static void scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, GrayQueue *queue); -static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue); +static void scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, ScanObjectFunc scan_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue); static void scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeReadyEntry *list, GrayQueue *queue); static void report_finalizer_roots (void); static void report_registered_roots (void); -static void find_pinning_ref_from_thread (char *obj, size_t size); -static void update_current_thread_stack (void *start); -static void collect_bridge_objects (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue); -static void finalize_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, GrayQueue *queue); -static void process_fin_stage_entries (void); -static void null_link_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end, int generation, gboolean before_finalization, GrayQueue *queue); -static void null_links_for_domain (MonoDomain *domain, int generation); -static void remove_finalizers_for_domain (MonoDomain *domain, int generation); -static void process_dislink_stage_entries (void); static void pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue); -static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue); +static int pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue, gboolean only_enqueue); static void finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue *queue); -static void mono_gc_register_disappearing_link (MonoObject *obj, void **link, gboolean track, gboolean in_gc); -static gboolean mono_gc_is_critical_method (MonoMethod *method); - void mono_gc_scan_for_specific_ref (MonoObject *key, gboolean precise); @@ -611,15 +554,51 @@ static GrayQueue gray_queue; static SgenRemeberedSet remset; +/* The gray queue to use from the main collection thread. */ +static SgenGrayQueue* +sgen_workers_get_main_thread_queue (void) +{ + if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) + return sgen_workers_get_distribute_gray_queue (); + return &gray_queue; +} -#define WORKERS_DISTRIBUTE_GRAY_QUEUE (sgen_collection_is_parallel () ? sgen_workers_get_distribute_gray_queue () : &gray_queue) +#define WORKERS_DISTRIBUTE_GRAY_QUEUE (sgen_workers_get_main_thread_queue ()) +/* + * The gray queue a worker job must use. If we're not parallel or + * concurrent, we use the main gray queue. + */ static SgenGrayQueue* sgen_workers_get_job_gray_queue (WorkerData *worker_data) { return worker_data ? &worker_data->private_gray_queue : WORKERS_DISTRIBUTE_GRAY_QUEUE; } +static LOCK_DECLARE (workers_distribute_gray_queue_mutex); + +void +sgen_remember_major_object_for_concurrent_mark (char *obj) +{ + gboolean need_lock = current_collection_generation != GENERATION_NURSERY; + + if (!major_collector.is_concurrent) + return; + + g_assert (current_collection_generation == GENERATION_NURSERY || current_collection_generation == -1); + + if (!concurrent_collection_in_progress) + return; + + if (need_lock) + mono_mutex_lock (&workers_distribute_gray_queue_mutex); + + sgen_gray_object_enqueue (sgen_workers_get_distribute_gray_queue (), obj); + + if (need_lock) + mono_mutex_unlock (&workers_distribute_gray_queue_mutex); +} + static gboolean is_xdomain_ref_allowed (gpointer *ptr, char *obj, MonoDomain *domain) { @@ -889,7 +868,7 @@ static gboolean need_remove_object_for_domain (char *start, MonoDomain *domain) { if (mono_object_domain (start) == domain) { - DEBUG (4, fprintf (gc_debug_file, "Need to cleanup object %p\n", start)); + SGEN_LOG (4, "Need to cleanup object %p", start); binary_protocol_cleanup (start, (gpointer)LOAD_VTABLE (start), safe_object_get_size ((MonoObject*)start)); return TRUE; } @@ -910,8 +889,7 @@ process_object_for_domain_clearing (char *start, MonoDomain *domain) /* The server could already have been zeroed out, so we need to check for that, too. */ if (server && (!LOAD_VTABLE (server) || mono_object_domain (server) == domain)) { - DEBUG (4, fprintf (gc_debug_file, "Cleaning up remote pointer in %p to object %p\n", - start, server)); + SGEN_LOG (4, "Cleaning up remote pointer in %p to object %p", start, server); ((MonoRealProxy*)start)->unwrapped_server = NULL; } } @@ -993,7 +971,7 @@ check_for_xdomain_refs (void) major_collector.iterate_objects (TRUE, TRUE, (IterateObjectCallbackFunc)scan_object_for_xdomain_refs, NULL); for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) - scan_object_for_xdomain_refs (bigobj->data, bigobj->size, NULL); + scan_object_for_xdomain_refs (bigobj->data, sgen_los_object_size (bigobj), NULL); } static gboolean @@ -1007,7 +985,7 @@ clear_domain_process_object (char *obj, MonoDomain *domain) if (remove && ((MonoObject*)obj)->synchronisation) { void **dislink = mono_monitor_get_object_monitor_weak_link ((MonoObject*)obj); if (dislink) - mono_gc_register_disappearing_link (NULL, dislink, FALSE, TRUE); + sgen_register_disappearing_link (NULL, dislink, FALSE, TRUE); } return remove; @@ -1057,8 +1035,8 @@ mono_gc_clear_domain (MonoDomain * domain) LOCK_GC; - process_fin_stage_entries (); - process_dislink_stage_entries (); + sgen_process_fin_stage_entries (); + sgen_process_dislink_stage_entries (); sgen_clear_nursery_fragments (); @@ -1073,10 +1051,10 @@ mono_gc_clear_domain (MonoDomain * domain) null_ephemerons_for_domain (domain); for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i) - null_links_for_domain (domain, i); + sgen_null_links_for_domain (domain, i); for (i = GENERATION_NURSERY; i < GENERATION_MAX; ++i) - remove_finalizers_for_domain (domain, i); + sgen_remove_finalizers_for_domain (domain, i); sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, (IterateObjectCallbackFunc)clear_domain_process_minor_object_callback, domain, FALSE); @@ -1101,8 +1079,7 @@ mono_gc_clear_domain (MonoDomain * domain) else los_object_list = bigobj->next; bigobj = bigobj->next; - DEBUG (4, fprintf (gc_debug_file, "Freeing large object %p\n", - bigobj->data)); + SGEN_LOG (4, "Freeing large object %p", bigobj->data); sgen_los_free_object (to_free); continue; } @@ -1143,17 +1120,16 @@ sgen_add_to_global_remset (gpointer ptr) * usage. */ gboolean -sgen_drain_gray_stack (GrayQueue *queue, int max_objs) +sgen_drain_gray_stack (GrayQueue *queue, ScanObjectFunc scan_func, int max_objs) { char *obj; - ScanObjectFunc scan_func = current_object_ops.scan_object; if (max_objs == -1) { for (;;) { GRAY_OBJECT_DEQUEUE (queue, obj); if (!obj) return TRUE; - DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj))); + SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj)); scan_func (obj, queue); } } else { @@ -1164,7 +1140,7 @@ sgen_drain_gray_stack (GrayQueue *queue, int max_objs) GRAY_OBJECT_DEQUEUE (queue, obj); if (!obj) return TRUE; - DEBUG (9, fprintf (gc_debug_file, "Precise gray object scan %p (%s)\n", obj, safe_name (obj))); + SGEN_LOG (9, "Precise gray object scan %p (%s)", obj, safe_name (obj)); scan_func (obj, queue); } } while (max_objs < 0); @@ -1180,7 +1156,7 @@ sgen_drain_gray_stack (GrayQueue *queue, int max_objs) * pinned objects. Return the number of pinned objects. */ static int -pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue) +pin_objects_from_addresses (GCMemSection *section, void **start, void **end, void *start_nursery, void *end_nursery, GrayQueue *queue, gboolean only_enqueue) { void *last = NULL; int count = 0; @@ -1197,7 +1173,7 @@ pin_objects_from_addresses (GCMemSection *section, void **start, void **end, voi addr = *start; /* the range check should be reduntant */ if (addr != last && addr >= start_nursery && addr < end_nursery) { - DEBUG (5, fprintf (gc_debug_file, "Considering pinning addr %p\n", addr)); + SGEN_LOG (5, "Considering pinning addr %p", addr); /* multiple pointers to the same object */ if (addr >= last_obj && (char*)addr < (char*)last_obj + last_obj_size) { start++; @@ -1241,16 +1217,17 @@ pin_objects_from_addresses (GCMemSection *section, void **start, void **end, voi if (((MonoObject*)last_obj)->synchronisation == GINT_TO_POINTER (-1)) { /* Marks the beginning of a nursery fragment, skip */ } else { - DEBUG (8, fprintf (gc_debug_file, "Pinned try match %p (%s), size %zd\n", last_obj, safe_name (last_obj), last_obj_size)); + SGEN_LOG (8, "Pinned try match %p (%s), size %zd", last_obj, safe_name (last_obj), last_obj_size); if (addr >= search_start && (char*)addr < (char*)last_obj + last_obj_size) { - DEBUG (4, fprintf (gc_debug_file, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count)); + SGEN_LOG (4, "Pinned object %p, vtable %p (%s), count %d\n", search_start, *(void**)search_start, safe_name (search_start), count); binary_protocol_pin (search_start, (gpointer)LOAD_VTABLE (search_start), safe_object_get_size (search_start)); - if (MONO_GC_OBJ_PINNED_ENABLED ()) { + if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) { int gen = sgen_ptr_in_nursery (search_start) ? GENERATION_NURSERY : GENERATION_OLD; MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (search_start); - MONO_GC_OBJ_PINNED (search_start, sgen_safe_object_get_size (search_start), vt->klass->name_space, vt->klass->name, gen); + MONO_GC_OBJ_PINNED ((mword)search_start, sgen_safe_object_get_size (search_start), vt->klass->name_space, vt->klass->name, gen); } - pin_object (search_start); + if (!only_enqueue) + pin_object (search_start); GRAY_OBJECT_ENQUEUE (queue, search_start); if (G_UNLIKELY (do_pin_stats)) sgen_pin_stats_register_object (search_start, last_obj_size); @@ -1282,14 +1259,14 @@ pin_objects_from_addresses (GCMemSection *section, void **start, void **end, voi } void -sgen_pin_objects_in_section (GCMemSection *section, GrayQueue *queue) +sgen_pin_objects_in_section (GCMemSection *section, GrayQueue *queue, gboolean only_enqueue) { int num_entries = section->pin_queue_num_entries; if (num_entries) { void **start = section->pin_queue_start; int reduced_to; reduced_to = pin_objects_from_addresses (section, start, start + num_entries, - section->data, section->next_data, queue); + section->data, section->next_data, queue, only_enqueue); section->pin_queue_num_entries = reduced_to; if (!reduced_to) section->pin_queue_start = NULL; @@ -1300,6 +1277,8 @@ sgen_pin_objects_in_section (GCMemSection *section, GrayQueue *queue) void sgen_pin_object (void *object, GrayQueue *queue) { + g_assert (!concurrent_collection_in_progress); + if (sgen_collection_is_parallel ()) { LOCK_PIN_QUEUE; /*object arrives pinned*/ @@ -1315,10 +1294,10 @@ sgen_pin_object (void *object, GrayQueue *queue) } GRAY_OBJECT_ENQUEUE (queue, object); binary_protocol_pin (object, (gpointer)LOAD_VTABLE (object), safe_object_get_size (object)); - if (MONO_GC_OBJ_PINNED_ENABLED ()) { + if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) { int gen = sgen_ptr_in_nursery (object) ? GENERATION_NURSERY : GENERATION_OLD; MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (object); - MONO_GC_OBJ_PINNED (object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen); + MONO_GC_OBJ_PINNED ((mword)object, sgen_safe_object_get_size (object), vt->klass->name_space, vt->klass->name, gen); } } @@ -1412,6 +1391,11 @@ static void conservatively_pin_objects_from (void **start, void **end, void *start_nursery, void *end_nursery, int pin_type) { int count = 0; + +#ifdef VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE + VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE (start, (char*)end - (char*)start); +#endif + while (start < end) { if (*start >= start_nursery && *start < end_nursery) { /* @@ -1433,43 +1417,20 @@ conservatively_pin_objects_from (void **start, void **end, void *start_nursery, */ mword addr = (mword)*start; addr &= ~(ALLOC_ALIGN - 1); - if (addr >= (mword)start_nursery && addr < (mword)end_nursery) + if (addr >= (mword)start_nursery && addr < (mword)end_nursery) { + SGEN_LOG (6, "Pinning address %p from %p", (void*)addr, start); sgen_pin_stage_ptr ((void*)addr); + count++; + } if (G_UNLIKELY (do_pin_stats)) { if (ptr_in_nursery ((void*)addr)) sgen_pin_stats_register_address ((char*)addr, pin_type); } - DEBUG (6, if (count) fprintf (gc_debug_file, "Pinning address %p from %p\n", (void*)addr, start)); - count++; } start++; } - DEBUG (7, if (count) fprintf (gc_debug_file, "found %d potential pinned heap pointers\n", count)); -} - -/* - * Debugging function: find in the conservative roots where @obj is being pinned. - */ -static G_GNUC_UNUSED void -find_pinning_reference (char *obj, size_t size) -{ - char **start; - RootRecord *root; - char *endobj = obj + size; - - SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_NORMAL], start, root) { - /* if desc is non-null it has precise info */ - if (!root->root_desc) { - while (start < (char**)root->end_root) { - if (*start >= obj && *start < endobj) { - DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in pinned roots %p-%p\n", obj, start, root->end_root)); - } - start++; - } - } - } SGEN_HASH_TABLE_FOREACH_END; - - find_pinning_ref_from_thread (obj, size); + if (count) + SGEN_LOG (7, "found %d potential pinned heap pointers", count); } /* @@ -1482,10 +1443,10 @@ pin_from_roots (void *start_nursery, void *end_nursery, GrayQueue *queue) { void **start_root; RootRecord *root; - DEBUG (2, fprintf (gc_debug_file, "Scanning pinned roots (%d bytes, %d/%d entries)\n", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries)); + SGEN_LOG (2, "Scanning pinned roots (%d bytes, %d/%d entries)", (int)roots_size, roots_hash [ROOT_TYPE_NORMAL].num_entries, roots_hash [ROOT_TYPE_PINNED].num_entries); /* objects pinned from the API are inside these roots */ SGEN_HASH_TABLE_FOREACH (&roots_hash [ROOT_TYPE_PINNED], start_root, root) { - DEBUG (6, fprintf (gc_debug_file, "Pinned roots %p-%p\n", start_root, root->end_root)); + SGEN_LOG (6, "Pinned roots %p-%p", start_root, root->end_root); conservatively_pin_objects_from (start_root, (void**)root->end_root, start_nursery, end_nursery, PIN_TYPE_OTHER); } SGEN_HASH_TABLE_FOREACH_END; /* now deal with the thread stacks @@ -1534,7 +1495,7 @@ single_arg_user_copy_or_mark (void **obj) * This function is not thread-safe! */ static void -precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc, GrayQueue *queue) +precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, ScanObjectFunc scan_func, void** start_root, void** end_root, char* n_start, char *n_end, mword desc, GrayQueue *queue) { switch (desc & ROOT_DESC_TYPE_MASK) { case ROOT_DESC_BITMAP: @@ -1542,8 +1503,8 @@ precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, while (desc) { if ((desc & 1) && *start_root) { copy_func (start_root, queue); - DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", start_root, *start_root)); - sgen_drain_gray_stack (queue, -1); + SGEN_LOG (9, "Overwrote root at %p with %p", start_root, *start_root); + sgen_drain_gray_stack (queue, scan_func, -1); } desc >>= 1; start_root++; @@ -1560,8 +1521,8 @@ precisely_scan_objects_from (CopyOrMarkObjectFunc copy_func, void** start_root, while (bmap) { if ((bmap & 1) && *objptr) { copy_func (objptr, queue); - DEBUG (9, fprintf (gc_debug_file, "Overwrote root at %p with %p\n", objptr, *objptr)); - sgen_drain_gray_stack (queue, -1); + SGEN_LOG (9, "Overwrote root at %p with %p", objptr, *objptr); + sgen_drain_gray_stack (queue, scan_func, -1); } bmap >>= 1; ++objptr; @@ -1624,7 +1585,7 @@ alloc_nursery (void) if (nursery_section) return; - DEBUG (2, fprintf (gc_debug_file, "Allocating nursery size: %lu\n", (unsigned long)sgen_nursery_size)); + SGEN_LOG (2, "Allocating nursery size: %lu", (unsigned long)sgen_nursery_size); /* later we will alloc a larger area for the nursery but only activate * what we need. The rest will be used as expansion if we have too many pinned * objects in the existing nursery. @@ -1643,7 +1604,7 @@ alloc_nursery (void) data = major_collector.alloc_heap (alloc_size, 0, DEFAULT_NURSERY_BITS); #endif sgen_update_heap_boundaries ((mword)data, (mword)(data + sgen_nursery_size)); - DEBUG (4, fprintf (gc_debug_file, "Expanding nursery size (%p-%p): %lu, total: %lu\n", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ())); + SGEN_LOG (4, "Expanding nursery size (%p-%p): %lu, total: %lu", data, data + alloc_size, (unsigned long)sgen_nursery_size, (unsigned long)mono_gc_get_heap_size ()); section->data = section->next_data = data; section->size = alloc_size; section->end_data = data + sgen_nursery_size; @@ -1691,7 +1652,7 @@ mono_gc_precise_stack_mark_enabled (void) FILE * mono_gc_get_logfile (void) { - return sgen_get_logfile (); + return gc_debug_file; } static void @@ -1779,7 +1740,7 @@ report_registered_roots_by_type (int root_type) RootRecord *root; report.count = 0; SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) { - DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", start_root, root->end_root, (void*)root->root_desc)); + SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc); precisely_report_roots_from (&report, start_root, (void**)root->end_root, root->root_desc); } SGEN_HASH_TABLE_FOREACH_END; notify_gc_roots (&report); @@ -1800,7 +1761,7 @@ scan_finalizer_entries (CopyOrMarkObjectFunc copy_func, FinalizeReadyEntry *list for (fin = list; fin; fin = fin->next) { if (!fin->object) continue; - DEBUG (5, fprintf (gc_debug_file, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object))); + SGEN_LOG (5, "Scan of fin ready object: %p (%s)\n", fin->object, safe_name (fin->object)); copy_func (&fin->object, queue); } } @@ -1815,17 +1776,10 @@ generation_name (int generation) } } - -static void -stw_bridge_process (void) -{ - sgen_bridge_processing_stw_step (); -} - -static void -bridge_process (void) +const char* +sgen_generation_name (int generation) { - sgen_bridge_processing_finish (); + return generation_name (generation); } SgenObjectOperations * @@ -1841,6 +1795,7 @@ finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue * TV_DECLARE (btv); int done_with_ephemerons, ephemeron_rounds = 0; CopyOrMarkObjectFunc copy_func = current_object_ops.copy_or_mark_object; + ScanObjectFunc scan_func = current_object_ops.scan_object; /* * We copied all the reachable objects. Now it's the time to copy @@ -1855,9 +1810,9 @@ finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue * * To achieve better cache locality and cache usage, we drain the gray stack * frequently, after each object is copied, and just finish the work here. */ - sgen_drain_gray_stack (queue, -1); + sgen_drain_gray_stack (queue, scan_func, -1); TV_GETTIME (atv); - DEBUG (2, fprintf (gc_debug_file, "%s generation done\n", generation_name (generation))); + SGEN_LOG (2, "%s generation done", generation_name (generation)); /* Reset bridge data, we might have lingering data from a previous collection if this is a major @@ -1876,7 +1831,7 @@ finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue * done_with_ephemerons = 0; do { done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr, queue); - sgen_drain_gray_stack (queue, -1); + sgen_drain_gray_stack (queue, scan_func, -1); ++ephemeron_rounds; } while (!done_with_ephemerons); @@ -1885,24 +1840,24 @@ finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue * sgen_scan_togglerefs (copy_func, sgen_get_nursery_start (), sgen_get_nursery_end (), queue); if (sgen_need_bridge_processing ()) { - collect_bridge_objects (copy_func, start_addr, end_addr, generation, queue); + sgen_collect_bridge_objects (copy_func, start_addr, end_addr, generation, queue); if (generation == GENERATION_OLD) - collect_bridge_objects (copy_func, sgen_get_nursery_start (), sgen_get_nursery_end (), GENERATION_NURSERY, queue); + sgen_collect_bridge_objects (copy_func, sgen_get_nursery_start (), sgen_get_nursery_end (), GENERATION_NURSERY, queue); } /* Make sure we drain the gray stack before processing disappearing links and finalizers. If we don't make sure it is empty we might wrongly see a live object as dead. */ - sgen_drain_gray_stack (queue, -1); + sgen_drain_gray_stack (queue, scan_func, -1); /* We must clear weak links that don't track resurrection before processing object ready for finalization so they can be cleared before that. */ - null_link_in_range (copy_func, start_addr, end_addr, generation, TRUE, queue); + sgen_null_link_in_range (copy_func, start_addr, end_addr, generation, TRUE, queue); if (generation == GENERATION_OLD) - null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, TRUE, queue); + sgen_null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, TRUE, queue); /* walk the finalization queue and move also the objects that need to be @@ -1910,12 +1865,12 @@ finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue * * on are also not reclaimed. As with the roots above, only objects in the nursery * are marked/copied. */ - finalize_in_range (copy_func, start_addr, end_addr, generation, queue); + sgen_finalize_in_range (copy_func, start_addr, end_addr, generation, queue); if (generation == GENERATION_OLD) - finalize_in_range (copy_func, sgen_get_nursery_start (), sgen_get_nursery_end (), GENERATION_NURSERY, queue); + sgen_finalize_in_range (copy_func, sgen_get_nursery_start (), sgen_get_nursery_end (), GENERATION_NURSERY, queue); /* drain the new stack that might have been created */ - DEBUG (6, fprintf (gc_debug_file, "Precise scan of gray area post fin\n")); - sgen_drain_gray_stack (queue, -1); + SGEN_LOG (6, "Precise scan of gray area post fin"); + sgen_drain_gray_stack (queue, scan_func, -1); /* * This must be done again after processing finalizable objects since CWL slots are cleared only after the key is finalized. @@ -1923,7 +1878,7 @@ finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue * done_with_ephemerons = 0; do { done_with_ephemerons = mark_ephemerons_in_range (copy_func, start_addr, end_addr, queue); - sgen_drain_gray_stack (queue, -1); + sgen_drain_gray_stack (queue, scan_func, -1); ++ephemeron_rounds; } while (!done_with_ephemerons); @@ -1934,7 +1889,7 @@ finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue * clear_unreachable_ephemerons (copy_func, start_addr, end_addr, queue); TV_GETTIME (btv); - DEBUG (2, fprintf (gc_debug_file, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron roundss\n", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds)); + SGEN_LOG (2, "Finalize queue handling scan for %s generation: %d usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds); /* * handle disappearing links @@ -1946,12 +1901,12 @@ finish_gray_stack (char *start_addr, char *end_addr, int generation, GrayQueue * */ g_assert (sgen_gray_object_queue_is_empty (queue)); for (;;) { - null_link_in_range (copy_func, start_addr, end_addr, generation, FALSE, queue); + sgen_null_link_in_range (copy_func, start_addr, end_addr, generation, FALSE, queue); if (generation == GENERATION_OLD) - null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, FALSE, queue); + sgen_null_link_in_range (copy_func, start_addr, end_addr, GENERATION_NURSERY, FALSE, queue); if (sgen_gray_object_queue_is_empty (queue)) break; - sgen_drain_gray_stack (queue, -1); + sgen_drain_gray_stack (queue, scan_func, -1); } g_assert (sgen_gray_object_queue_is_empty (queue)); @@ -1979,13 +1934,13 @@ check_scan_starts (void) } static void -scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue) +scan_from_registered_roots (CopyOrMarkObjectFunc copy_func, ScanObjectFunc scan_func, char *addr_start, char *addr_end, int root_type, GrayQueue *queue) { void **start_root; RootRecord *root; SGEN_HASH_TABLE_FOREACH (&roots_hash [root_type], start_root, root) { - DEBUG (6, fprintf (gc_debug_file, "Precise root scan %p-%p (desc: %p)\n", start_root, root->end_root, (void*)root->root_desc)); - precisely_scan_objects_from (copy_func, start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, queue); + SGEN_LOG (6, "Precise root scan %p-%p (desc: %p)", start_root, root->end_root, (void*)root->root_desc); + precisely_scan_objects_from (copy_func, scan_func, start_root, (void**)root->end_root, addr_start, addr_end, root->root_desc, queue); } SGEN_HASH_TABLE_FOREACH_END; } @@ -2223,6 +2178,25 @@ sgen_collection_is_parallel (void) } } +gboolean +sgen_collection_is_concurrent (void) +{ + switch (current_collection_generation) { + case GENERATION_NURSERY: + return FALSE; + case GENERATION_OLD: + return major_collector.is_concurrent; + default: + g_error ("Invalid current generation %d", current_collection_generation); + } +} + +gboolean +sgen_concurrent_collection_in_progress (void) +{ + return concurrent_collection_in_progress; +} + typedef struct { char *heap_start; @@ -2235,11 +2209,13 @@ job_finish_remembered_set_scan (WorkerData *worker_data, void *job_data_untyped) FinishRememberedSetScanJobData *job_data = job_data_untyped; remset.finish_scan_remsets (job_data->heap_start, job_data->heap_end, sgen_workers_get_job_gray_queue (worker_data)); + sgen_free_internal_dynamic (job_data, sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA); } typedef struct { - CopyOrMarkObjectFunc func; + CopyOrMarkObjectFunc copy_or_mark_func; + ScanObjectFunc scan_func; char *heap_start; char *heap_end; int root_type; @@ -2250,10 +2226,11 @@ job_scan_from_registered_roots (WorkerData *worker_data, void *job_data_untyped) { ScanFromRegisteredRootsJobData *job_data = job_data_untyped; - scan_from_registered_roots (job_data->func, + scan_from_registered_roots (job_data->copy_or_mark_func, job_data->scan_func, job_data->heap_start, job_data->heap_end, job_data->root_type, sgen_workers_get_job_gray_queue (worker_data)); + sgen_free_internal_dynamic (job_data, sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA); } typedef struct @@ -2269,6 +2246,7 @@ job_scan_thread_data (WorkerData *worker_data, void *job_data_untyped) scan_thread_data (job_data->heap_start, job_data->heap_end, TRUE, sgen_workers_get_job_gray_queue (worker_data)); + sgen_free_internal_dynamic (job_data, sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA); } typedef struct @@ -2284,6 +2262,21 @@ job_scan_finalizer_entries (WorkerData *worker_data, void *job_data_untyped) scan_finalizer_entries (current_object_ops.copy_or_mark_object, job_data->list, sgen_workers_get_job_gray_queue (worker_data)); + sgen_free_internal_dynamic (job_data, sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA); +} + +static void +job_scan_major_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped) +{ + g_assert (concurrent_collection_in_progress); + major_collector.scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data)); +} + +static void +job_scan_los_mod_union_cardtable (WorkerData *worker_data, void *job_data_untyped) +{ + g_assert (concurrent_collection_in_progress); + sgen_los_scan_card_table (TRUE, sgen_workers_get_job_gray_queue (worker_data)); } static void @@ -2294,7 +2287,7 @@ verify_scan_starts (char *start, char *end) for (i = 0; i < nursery_section->num_scan_start; ++i) { char *addr = nursery_section->scan_starts [i]; if (addr > start && addr < end) - fprintf (gc_debug_file, "NFC-BAD SCAN START [%d] %p for obj [%p %p]\n", i, addr, start, end); + SGEN_LOG (1, "NFC-BAD SCAN START [%d] %p for obj [%p %p]", i, addr, start, end); } } @@ -2321,22 +2314,65 @@ verify_nursery (void) } if (object_is_forwarded (cur)) - fprintf (gc_debug_file, "FORWARDED OBJ %p\n", cur); + SGEN_LOG (1, "FORWARDED OBJ %p", cur); else if (object_is_pinned (cur)) - fprintf (gc_debug_file, "PINNED OBJ %p\n", cur); + SGEN_LOG (1, "PINNED OBJ %p", cur); ss = safe_object_get_size ((MonoObject*)cur); size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur)); verify_scan_starts (cur, cur + size); if (do_dump_nursery_content) { if (cur > hole_start) - fprintf (gc_debug_file, "HOLE [%p %p %d]\n", hole_start, cur, (int)(cur - hole_start)); - fprintf (gc_debug_file, "OBJ [%p %p %d %d %s %d]\n", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ()); + SGEN_LOG (1, "HOLE [%p %p %d]", hole_start, cur, (int)(cur - hole_start)); + SGEN_LOG (1, "OBJ [%p %p %d %d %s %d]", cur, cur + size, (int)size, (int)ss, sgen_safe_name ((MonoObject*)cur), (gpointer)LOAD_VTABLE (cur) == sgen_get_array_fill_vtable ()); } cur += size; hole_start = cur; } - fflush (gc_debug_file); +} + +/* + * Checks that no objects in the nursery are fowarded or pinned. This + * is a precondition to restarting the mutator while doing a + * concurrent collection. Note that we don't clear fragments because + * we depend on that having happened earlier. + */ +static void +check_nursery_is_clean (void) +{ + char *start, *end, *cur; + + start = cur = sgen_get_nursery_start (); + end = sgen_get_nursery_end (); + + while (cur < end) { + size_t ss, size; + + if (!*(void**)cur) { + cur += sizeof (void*); + continue; + } + + g_assert (!object_is_forwarded (cur)); + g_assert (!object_is_pinned (cur)); + + ss = safe_object_get_size ((MonoObject*)cur); + size = ALIGN_UP (safe_object_get_size ((MonoObject*)cur)); + verify_scan_starts (cur, cur + size); + + cur += size; + } +} + +static void +init_gray_queue (void) +{ + if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) { + sgen_gray_object_queue_init_invalid (&gray_queue); + sgen_workers_init_distribute_gray_queue (); + } else { + sgen_gray_object_queue_init (&gray_queue); + } } /* @@ -2349,10 +2385,10 @@ collect_nursery (void) gboolean needs_major; size_t max_garbage_amount; char *nursery_next; - FinishRememberedSetScanJobData frssjd; - ScanFromRegisteredRootsJobData scrrjd_normal, scrrjd_wbarrier; - ScanFinalizerEntriesJobData sfejd_fin_ready, sfejd_critical_fin; - ScanThreadDataJobData stdjd; + FinishRememberedSetScanJobData *frssjd; + ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier; + ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin; + ScanThreadDataJobData *stdjd; mword fragment_total; TV_DECLARE (all_atv); TV_DECLARE (all_btv); @@ -2366,7 +2402,9 @@ collect_nursery (void) verify_nursery (); +#ifndef DISABLE_PERFCOUNTERS mono_perfcounters->gc_collections0++; +#endif current_collection_generation = GENERATION_NURSERY; if (sgen_collection_is_parallel ()) @@ -2376,7 +2414,7 @@ collect_nursery (void) reset_pinned_from_failed_allocation (); - binary_protocol_collection (GENERATION_NURSERY); + binary_protocol_collection (stat_minor_gcs, GENERATION_NURSERY); check_scan_starts (); sgen_nursery_alloc_prepare_for_minor (); @@ -2387,7 +2425,7 @@ collect_nursery (void) /* FIXME: optimize later to use the higher address where an object can be present */ nursery_next = MAX (nursery_next, sgen_get_nursery_end ()); - DEBUG (1, fprintf (gc_debug_file, "Start nursery collection %d %p-%p, size: %d\n", stat_minor_gcs, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ()))); + SGEN_LOG (1, "Start nursery collection %d %p-%p, size: %d", stat_minor_gcs, sgen_get_nursery_start (), nursery_next, (int)(nursery_next - sgen_get_nursery_start ())); max_garbage_amount = nursery_next - sgen_get_nursery_start (); g_assert (nursery_section->size >= max_garbage_amount); @@ -2409,8 +2447,7 @@ collect_nursery (void) sgen_memgov_minor_collection_start (); - sgen_gray_object_queue_init (&gray_queue); - sgen_workers_init_distribute_gray_queue (); + init_gray_queue (); stat_minor_gcs++; gc_stats.minor_gc_count ++; @@ -2418,8 +2455,8 @@ collect_nursery (void) if (remset.prepare_for_minor_collection) remset.prepare_for_minor_collection (); - process_fin_stage_entries (); - process_dislink_stage_entries (); + sgen_process_fin_stage_entries (); + sgen_process_dislink_stage_entries (); /* pin from pinned handles */ sgen_init_pinning (); @@ -2428,13 +2465,13 @@ collect_nursery (void) /* identify pinned objects */ sgen_optimize_pin_queue (0); sgen_pinning_setup_section (nursery_section); - sgen_pin_objects_in_section (nursery_section, WORKERS_DISTRIBUTE_GRAY_QUEUE); + sgen_pin_objects_in_section (nursery_section, WORKERS_DISTRIBUTE_GRAY_QUEUE, FALSE); sgen_pinning_trim_queue_to_section (nursery_section); TV_GETTIME (atv); time_minor_pinning += TV_ELAPSED (btv, atv); - DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", sgen_get_pinned_count (), TV_ELAPSED (btv, atv))); - DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", sgen_get_pinned_count ())); + SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv)); + SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ()); if (whole_heap_check_before_collection) sgen_check_whole_heap (); @@ -2452,17 +2489,18 @@ collect_nursery (void) sgen_workers_start_marking (); - frssjd.heap_start = sgen_get_nursery_start (); - frssjd.heap_end = nursery_next; - sgen_workers_enqueue_job (job_finish_remembered_set_scan, &frssjd); + frssjd = sgen_alloc_internal_dynamic (sizeof (FinishRememberedSetScanJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + frssjd->heap_start = sgen_get_nursery_start (); + frssjd->heap_end = nursery_next; + sgen_workers_enqueue_job (job_finish_remembered_set_scan, frssjd); /* we don't have complete write barrier yet, so we scan all the old generation sections */ TV_GETTIME (btv); time_minor_scan_remsets += TV_ELAPSED (atv, btv); - DEBUG (2, fprintf (gc_debug_file, "Old generation scan: %d usecs\n", TV_ELAPSED (atv, btv))); + SGEN_LOG (2, "Old generation scan: %d usecs", TV_ELAPSED (atv, btv)); if (!sgen_collection_is_parallel ()) - sgen_drain_gray_stack (&gray_queue, -1); + sgen_drain_gray_stack (&gray_queue, current_object_ops.scan_object, -1); if (mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS) report_registered_roots (); @@ -2472,31 +2510,36 @@ collect_nursery (void) time_minor_scan_pinned += TV_ELAPSED (btv, atv); /* registered roots, this includes static fields */ - scrrjd_normal.func = current_object_ops.copy_or_mark_object; - scrrjd_normal.heap_start = sgen_get_nursery_start (); - scrrjd_normal.heap_end = nursery_next; - scrrjd_normal.root_type = ROOT_TYPE_NORMAL; - sgen_workers_enqueue_job (job_scan_from_registered_roots, &scrrjd_normal); - - scrrjd_wbarrier.func = current_object_ops.copy_or_mark_object; - scrrjd_wbarrier.heap_start = sgen_get_nursery_start (); - scrrjd_wbarrier.heap_end = nursery_next; - scrrjd_wbarrier.root_type = ROOT_TYPE_WBARRIER; - sgen_workers_enqueue_job (job_scan_from_registered_roots, &scrrjd_wbarrier); + scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object; + scrrjd_normal->scan_func = current_object_ops.scan_object; + scrrjd_normal->heap_start = sgen_get_nursery_start (); + scrrjd_normal->heap_end = nursery_next; + scrrjd_normal->root_type = ROOT_TYPE_NORMAL; + sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal); + + scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object; + scrrjd_wbarrier->scan_func = current_object_ops.scan_object; + scrrjd_wbarrier->heap_start = sgen_get_nursery_start (); + scrrjd_wbarrier->heap_end = nursery_next; + scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER; + sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier); TV_GETTIME (btv); time_minor_scan_registered_roots += TV_ELAPSED (atv, btv); /* thread data */ - stdjd.heap_start = sgen_get_nursery_start (); - stdjd.heap_end = nursery_next; - sgen_workers_enqueue_job (job_scan_thread_data, &stdjd); + stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + stdjd->heap_start = sgen_get_nursery_start (); + stdjd->heap_end = nursery_next; + sgen_workers_enqueue_job (job_scan_thread_data, stdjd); TV_GETTIME (atv); time_minor_scan_thread_data += TV_ELAPSED (btv, atv); btv = atv; - if (sgen_collection_is_parallel ()) { + if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) { while (!sgen_gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)) { sgen_workers_distribute_gray_queue_sections (); g_usleep (1000); @@ -2504,15 +2547,17 @@ collect_nursery (void) } sgen_workers_join (); - if (sgen_collection_is_parallel ()) + if (sgen_collection_is_parallel () || sgen_collection_is_concurrent ()) g_assert (sgen_gray_object_queue_is_empty (&gray_queue)); /* Scan the list of objects ready for finalization. If */ - sfejd_fin_ready.list = fin_ready_list; - sgen_workers_enqueue_job (job_scan_finalizer_entries, &sfejd_fin_ready); + sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + sfejd_fin_ready->list = fin_ready_list; + sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready); - sfejd_critical_fin.list = critical_fin_list; - sgen_workers_enqueue_job (job_scan_finalizer_entries, &sfejd_critical_fin); + sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + sfejd_critical_fin->list = critical_fin_list; + sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin); finish_gray_stack (sgen_get_nursery_start (), nursery_next, GENERATION_NURSERY, &gray_queue); TV_GETTIME (atv); @@ -2547,7 +2592,7 @@ collect_nursery (void) mono_profiler_gc_event (MONO_GC_EVENT_RECLAIM_END, 0); TV_GETTIME (btv); time_minor_fragment_creation += TV_ELAPSED (atv, btv); - DEBUG (2, fprintf (gc_debug_file, "Fragment creation: %d usecs, %lu bytes available\n", TV_ELAPSED (atv, btv), (unsigned long)fragment_total)); + SGEN_LOG (2, "Fragment creation: %d usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total); if (consistency_check_at_minor_collection) sgen_check_major_refs (); @@ -2563,7 +2608,7 @@ collect_nursery (void) /* prepare the pin queue for the next collection */ sgen_finish_pinning (); if (fin_ready_list || critical_fin_list) { - DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers)); + SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers); mono_gc_finalize_notify (); } sgen_pin_stats_reset (); @@ -2589,12 +2634,10 @@ collect_nursery (void) return needs_major; } -static gboolean -major_do_collection (const char *reason) +static void +major_copy_or_mark_from_roots (int *old_next_pin_slot, gboolean finish_up_concurrent_mark) { - LOSObject *bigobj, *prevbo; - TV_DECLARE (all_atv); - TV_DECLARE (all_btv); + LOSObject *bigobj; TV_DECLARE (atv); TV_DECLARE (btv); /* FIXME: only use these values for the precise scan @@ -2602,40 +2645,25 @@ major_do_collection (const char *reason) */ char *heap_start = NULL; char *heap_end = (char*)-1; - int old_next_pin_slot; - ScanFromRegisteredRootsJobData scrrjd_normal, scrrjd_wbarrier; - ScanThreadDataJobData stdjd; - ScanFinalizerEntriesJobData sfejd_fin_ready, sfejd_critical_fin; + gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS; + GCRootReport root_report = { 0 }; + ScanFromRegisteredRootsJobData *scrrjd_normal, *scrrjd_wbarrier; + ScanThreadDataJobData *stdjd; + ScanFinalizerEntriesJobData *sfejd_fin_ready, *sfejd_critical_fin; - MONO_GC_BEGIN (GENERATION_OLD); - - current_collection_generation = GENERATION_OLD; - mono_perfcounters->gc_collections1++; - - current_object_ops = major_collector.major_ops; - - reset_pinned_from_failed_allocation (); - - sgen_memgov_major_collection_start (); + if (major_collector.is_concurrent) { + /*This cleans up unused fragments */ + sgen_nursery_allocator_prepare_for_pinning (); - //count_ref_nonref_objs (); - //consistency_check (); - - binary_protocol_collection (GENERATION_OLD); - check_scan_starts (); + check_nursery_is_clean (); + } else { + /* The concurrent collector doesn't touch the nursery. */ + sgen_nursery_alloc_prepare_for_major (); + } - sgen_gray_object_queue_init (&gray_queue); - sgen_workers_init_distribute_gray_queue (); - sgen_nursery_alloc_prepare_for_major (); + init_gray_queue (); - degraded_mode = 0; - DEBUG (1, fprintf (gc_debug_file, "Start major collection %d\n", stat_major_gcs)); - stat_major_gcs++; - gc_stats.major_gc_count ++; - - /* world must be stopped already */ - TV_GETTIME (all_atv); - atv = all_atv; + TV_GETTIME (atv); /* Pinning depends on this */ sgen_clear_nursery_fragments (); @@ -2646,14 +2674,12 @@ major_do_collection (const char *reason) TV_GETTIME (btv); time_major_pre_collection_fragment_clear += TV_ELAPSED (atv, btv); - nursery_section->next_data = sgen_get_nursery_end (); + if (!sgen_collection_is_concurrent ()) + nursery_section->next_data = sgen_get_nursery_end (); /* we should also coalesce scanning from sections close to each other * and deal with pointers outside of the sections later. */ - if (major_collector.start_major_collection) - major_collector.start_major_collection (); - objects_pinned = 0; *major_collector.have_swept = FALSE; @@ -2662,15 +2688,17 @@ major_do_collection (const char *reason) check_for_xdomain_refs (); } - /* Remsets are not useful for a major collection */ - remset.prepare_for_major_collection (); + if (!finish_up_concurrent_mark) { + /* Remsets are not useful for a major collection */ + remset.prepare_for_major_collection (); + } - process_fin_stage_entries (); - process_dislink_stage_entries (); + sgen_process_fin_stage_entries (); + sgen_process_dislink_stage_entries (); TV_GETTIME (atv); sgen_init_pinning (); - DEBUG (6, fprintf (gc_debug_file, "Collecting pinned addresses\n")); + SGEN_LOG (6, "Collecting pinned addresses"); pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, WORKERS_DISTRIBUTE_GRAY_QUEUE); sgen_optimize_pin_queue (0); @@ -2686,45 +2714,47 @@ major_do_collection (const char *reason) * The second, destructive, pass is to reduce the section * areas to pointers to the actually pinned objects. */ - DEBUG (6, fprintf (gc_debug_file, "Pinning from sections\n")); + SGEN_LOG (6, "Pinning from sections"); /* first pass for the sections */ sgen_find_section_pin_queue_start_end (nursery_section); major_collector.find_pin_queue_start_ends (WORKERS_DISTRIBUTE_GRAY_QUEUE); /* identify possible pointers to the insize of large objects */ - DEBUG (6, fprintf (gc_debug_file, "Pinning from large objects\n")); + SGEN_LOG (6, "Pinning from large objects"); for (bigobj = los_object_list; bigobj; bigobj = bigobj->next) { int dummy; - gboolean profile_roots = mono_profiler_get_events () & MONO_PROFILE_GC_ROOTS; - GCRootReport report; - report.count = 0; - if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + bigobj->size, &dummy)) { - binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (bigobj->data)); - if (MONO_GC_OBJ_PINNED_ENABLED ()) { + if (sgen_find_optimized_pin_queue_area (bigobj->data, (char*)bigobj->data + sgen_los_object_size (bigobj), &dummy)) { + binary_protocol_pin (bigobj->data, (gpointer)LOAD_VTABLE (bigobj->data), safe_object_get_size (((MonoObject*)(bigobj->data)))); + if (G_UNLIKELY (MONO_GC_OBJ_PINNED_ENABLED ())) { MonoVTable *vt = (MonoVTable*)LOAD_VTABLE (bigobj->data); - MONO_GC_OBJ_PINNED (bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD); + MONO_GC_OBJ_PINNED ((mword)bigobj->data, sgen_safe_object_get_size ((MonoObject*)bigobj->data), vt->klass->name_space, vt->klass->name, GENERATION_OLD); + } + if (sgen_los_object_is_pinned (bigobj->data)) { + g_assert (finish_up_concurrent_mark); + continue; } - pin_object (bigobj->data); + sgen_los_pin_object (bigobj->data); /* FIXME: only enqueue if object has references */ GRAY_OBJECT_ENQUEUE (WORKERS_DISTRIBUTE_GRAY_QUEUE, bigobj->data); if (G_UNLIKELY (do_pin_stats)) sgen_pin_stats_register_object ((char*) bigobj->data, safe_object_get_size ((MonoObject*) bigobj->data)); - DEBUG (6, fprintf (gc_debug_file, "Marked large object %p (%s) size: %lu from roots\n", bigobj->data, safe_name (bigobj->data), (unsigned long)bigobj->size)); - + SGEN_LOG (6, "Marked large object %p (%s) size: %lu from roots", bigobj->data, safe_name (bigobj->data), (unsigned long)sgen_los_object_size (bigobj)); + if (profile_roots) - add_profile_gc_root (&report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0); + add_profile_gc_root (&root_report, bigobj->data, MONO_PROFILE_GC_ROOT_PINNING | MONO_PROFILE_GC_ROOT_MISC, 0); } - if (profile_roots) - notify_gc_roots (&report); } + if (profile_roots) + notify_gc_roots (&root_report); /* second pass for the sections */ - sgen_pin_objects_in_section (nursery_section, WORKERS_DISTRIBUTE_GRAY_QUEUE); + sgen_pin_objects_in_section (nursery_section, WORKERS_DISTRIBUTE_GRAY_QUEUE, concurrent_collection_in_progress); major_collector.pin_objects (WORKERS_DISTRIBUTE_GRAY_QUEUE); - old_next_pin_slot = sgen_get_pinned_count (); + if (old_next_pin_slot) + *old_next_pin_slot = sgen_get_pinned_count (); TV_GETTIME (btv); time_major_pinning += TV_ELAPSED (atv, btv); - DEBUG (2, fprintf (gc_debug_file, "Finding pinned pointers: %d in %d usecs\n", sgen_get_pinned_count (), TV_ELAPSED (atv, btv))); - DEBUG (4, fprintf (gc_debug_file, "Start scan with %d pinned objects\n", sgen_get_pinned_count ())); + SGEN_LOG (2, "Finding pinned pointers: %d in %d usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv)); + SGEN_LOG (4, "Start scan with %d pinned objects", sgen_get_pinned_count ()); major_collector.init_to_space (); @@ -2741,25 +2771,30 @@ major_do_collection (const char *reason) time_major_scan_pinned += TV_ELAPSED (btv, atv); /* registered roots, this includes static fields */ - scrrjd_normal.func = current_object_ops.copy_or_mark_object; - scrrjd_normal.heap_start = heap_start; - scrrjd_normal.heap_end = heap_end; - scrrjd_normal.root_type = ROOT_TYPE_NORMAL; - sgen_workers_enqueue_job (job_scan_from_registered_roots, &scrrjd_normal); - - scrrjd_wbarrier.func = current_object_ops.copy_or_mark_object; - scrrjd_wbarrier.heap_start = heap_start; - scrrjd_wbarrier.heap_end = heap_end; - scrrjd_wbarrier.root_type = ROOT_TYPE_WBARRIER; - sgen_workers_enqueue_job (job_scan_from_registered_roots, &scrrjd_wbarrier); + scrrjd_normal = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + scrrjd_normal->copy_or_mark_func = current_object_ops.copy_or_mark_object; + scrrjd_normal->scan_func = current_object_ops.scan_object; + scrrjd_normal->heap_start = heap_start; + scrrjd_normal->heap_end = heap_end; + scrrjd_normal->root_type = ROOT_TYPE_NORMAL; + sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_normal); + + scrrjd_wbarrier = sgen_alloc_internal_dynamic (sizeof (ScanFromRegisteredRootsJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + scrrjd_wbarrier->copy_or_mark_func = current_object_ops.copy_or_mark_object; + scrrjd_wbarrier->scan_func = current_object_ops.scan_object; + scrrjd_wbarrier->heap_start = heap_start; + scrrjd_wbarrier->heap_end = heap_end; + scrrjd_wbarrier->root_type = ROOT_TYPE_WBARRIER; + sgen_workers_enqueue_job (job_scan_from_registered_roots, scrrjd_wbarrier); TV_GETTIME (btv); time_major_scan_registered_roots += TV_ELAPSED (atv, btv); /* Threads */ - stdjd.heap_start = heap_start; - stdjd.heap_end = heap_end; - sgen_workers_enqueue_job (job_scan_thread_data, &stdjd); + stdjd = sgen_alloc_internal_dynamic (sizeof (ScanThreadDataJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + stdjd->heap_start = heap_start; + stdjd->heap_end = heap_end; + sgen_workers_enqueue_job (job_scan_thread_data, stdjd); TV_GETTIME (atv); time_major_scan_thread_data += TV_ELAPSED (btv, atv); @@ -2771,20 +2806,77 @@ major_do_collection (const char *reason) report_finalizer_roots (); /* scan the list of objects ready for finalization */ - sfejd_fin_ready.list = fin_ready_list; - sgen_workers_enqueue_job (job_scan_finalizer_entries, &sfejd_fin_ready); - - sfejd_critical_fin.list = critical_fin_list; - sgen_workers_enqueue_job (job_scan_finalizer_entries, &sfejd_critical_fin); + sfejd_fin_ready = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + sfejd_fin_ready->list = fin_ready_list; + sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_fin_ready); + + sfejd_critical_fin = sgen_alloc_internal_dynamic (sizeof (ScanFinalizerEntriesJobData), INTERNAL_MEM_WORKER_JOB_DATA, TRUE); + sfejd_critical_fin->list = critical_fin_list; + sgen_workers_enqueue_job (job_scan_finalizer_entries, sfejd_critical_fin); + + if (finish_up_concurrent_mark) { + /* Mod union card table */ + sgen_workers_enqueue_job (job_scan_major_mod_union_cardtable, NULL); + sgen_workers_enqueue_job (job_scan_los_mod_union_cardtable, NULL); + } TV_GETTIME (atv); time_major_scan_finalized += TV_ELAPSED (btv, atv); - DEBUG (2, fprintf (gc_debug_file, "Root scan: %d usecs\n", TV_ELAPSED (btv, atv))); + SGEN_LOG (2, "Root scan: %d usecs", TV_ELAPSED (btv, atv)); TV_GETTIME (btv); time_major_scan_big_objects += TV_ELAPSED (atv, btv); - if (major_collector.is_parallel) { + if (major_collector.is_concurrent) { + /* prepare the pin queue for the next collection */ + sgen_finish_pinning (); + + sgen_pin_stats_reset (); + + check_nursery_is_clean (); + } +} + +static void +major_start_collection (int *old_next_pin_slot) +{ + MONO_GC_BEGIN (GENERATION_OLD); + + current_collection_generation = GENERATION_OLD; +#ifndef DISABLE_PERFCOUNTERS + mono_perfcounters->gc_collections1++; +#endif + + if (major_collector.is_concurrent) + concurrent_collection_in_progress = TRUE; + + current_object_ops = major_collector.major_ops; + + reset_pinned_from_failed_allocation (); + + sgen_memgov_major_collection_start (); + + //count_ref_nonref_objs (); + //consistency_check (); + + binary_protocol_collection (stat_major_gcs, GENERATION_OLD); + check_scan_starts (); + + degraded_mode = 0; + SGEN_LOG (1, "Start major collection %d", stat_major_gcs); + stat_major_gcs++; + gc_stats.major_gc_count ++; + + if (major_collector.start_major_collection) + major_collector.start_major_collection (); + + major_copy_or_mark_from_roots (old_next_pin_slot, FALSE); +} + +static void +wait_for_workers_to_finish (void) +{ + if (major_collector.is_parallel || major_collector.is_concurrent) { while (!sgen_gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)) { sgen_workers_distribute_gray_queue_sections (); g_usleep (1000); @@ -2792,12 +2884,38 @@ major_do_collection (const char *reason) } sgen_workers_join (); + if (major_collector.is_parallel || major_collector.is_concurrent) + g_assert (sgen_gray_object_queue_is_empty (&gray_queue)); + #ifdef SGEN_DEBUG_INTERNAL_ALLOC main_gc_thread = NULL; #endif +} - if (major_collector.is_parallel) - g_assert (sgen_gray_object_queue_is_empty (&gray_queue)); +static void +major_finish_collection (const char *reason, int old_next_pin_slot) +{ + LOSObject *bigobj, *prevbo; + TV_DECLARE (atv); + TV_DECLARE (btv); + char *heap_start = NULL; + char *heap_end = (char*)-1; + + TV_GETTIME (btv); + + wait_for_workers_to_finish (); + + current_object_ops = major_collector.major_ops; + + if (major_collector.is_concurrent) { + major_collector.update_cardtable_mod_union (); + sgen_los_update_cardtable_mod_union (); + + major_copy_or_mark_from_roots (NULL, TRUE); + wait_for_workers_to_finish (); + + check_nursery_is_clean (); + } /* all the objects in the heap */ finish_gray_stack (heap_start, heap_end, GENERATION_OLD, &gray_queue); @@ -2813,6 +2931,8 @@ major_do_collection (const char *reason) sgen_workers_reset_data (); if (objects_pinned) { + g_assert (!major_collector.is_concurrent); + /*This is slow, but we just OOM'd*/ sgen_pin_queue_clear_discarded_entries (nursery_section, old_next_pin_slot); sgen_optimize_pin_queue (0); @@ -2826,9 +2946,10 @@ major_do_collection (const char *reason) /* sweep the big objects list */ prevbo = NULL; for (bigobj = los_object_list; bigobj;) { - if (object_is_pinned (bigobj->data)) { - unpin_object (bigobj->data); - sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + bigobj->size); + g_assert (!object_is_pinned (bigobj->data)); + if (sgen_los_object_is_pinned (bigobj->data)) { + sgen_los_unpin_object (bigobj->data); + sgen_update_heap_boundaries ((mword)bigobj->data, (mword)bigobj->data + sgen_los_object_size (bigobj)); } else { LOSObject *to_free; /* not referenced anywhere, so we can free it */ @@ -2858,33 +2979,33 @@ major_do_collection (const char *reason) TV_GETTIME (btv); time_major_sweep += TV_ELAPSED (atv, btv); - /* walk the pin_queue, build up the fragment list of free memory, unmark - * pinned objects as we go, memzero() the empty fragments so they are ready for the - * next allocations. - */ - if (!sgen_build_nursery_fragments (nursery_section, nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries)) - degraded_mode = 1; + if (!major_collector.is_concurrent) { + /* walk the pin_queue, build up the fragment list of free memory, unmark + * pinned objects as we go, memzero() the empty fragments so they are ready for the + * next allocations. + */ + if (!sgen_build_nursery_fragments (nursery_section, nursery_section->pin_queue_start, nursery_section->pin_queue_num_entries)) + degraded_mode = 1; - /* Clear TLABs for all threads */ - sgen_clear_tlabs (); + /* prepare the pin queue for the next collection */ + sgen_finish_pinning (); + + /* Clear TLABs for all threads */ + sgen_clear_tlabs (); + + sgen_pin_stats_reset (); + } TV_GETTIME (atv); time_major_fragment_creation += TV_ELAPSED (btv, atv); - TV_GETTIME (all_btv); - gc_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv); - if (heap_dump_file) dump_heap ("major", stat_major_gcs - 1, reason); - /* prepare the pin queue for the next collection */ - sgen_finish_pinning (); - if (fin_ready_list || critical_fin_list) { - DEBUG (4, fprintf (gc_debug_file, "Finalizer-thread wakeup: ready %d\n", num_ready_finalizers)); + SGEN_LOG (4, "Finalizer-thread wakeup: ready %d", num_ready_finalizers); mono_gc_finalize_notify (); } - sgen_pin_stats_reset (); g_assert (sgen_gray_object_queue_is_empty (&gray_queue)); @@ -2893,6 +3014,9 @@ major_do_collection (const char *reason) major_collector.finish_major_collection (); + if (major_collector.is_concurrent) + concurrent_collection_in_progress = FALSE; + check_scan_starts (); binary_protocol_flush_buffers (FALSE); @@ -2900,12 +3024,51 @@ major_do_collection (const char *reason) //consistency_check (); MONO_GC_END (GENERATION_OLD); +} + +static gboolean +major_do_collection (const char *reason) +{ + TV_DECLARE (all_atv); + TV_DECLARE (all_btv); + int old_next_pin_slot; + + /* world must be stopped already */ + TV_GETTIME (all_atv); + + major_start_collection (&old_next_pin_slot); + major_finish_collection (reason, old_next_pin_slot); + + TV_GETTIME (all_btv); + gc_stats.major_gc_time_usecs += TV_ELAPSED (all_atv, all_btv); return bytes_pinned_from_failed_allocation > 0; } static gboolean major_do_collection (const char *reason); +static void +major_start_concurrent_collection (const char *reason) +{ + // FIXME: store reason and pass it when finishing + major_start_collection (NULL); + + sgen_workers_distribute_gray_queue_sections (); + g_assert (sgen_gray_object_queue_is_empty (WORKERS_DISTRIBUTE_GRAY_QUEUE)); + + sgen_workers_wait_for_jobs (); + + current_collection_generation = -1; +} + +static void +major_finish_concurrent_collection (void) +{ + current_collection_generation = GENERATION_OLD; + major_finish_collection ("finishing", -1); + current_collection_generation = -1; +} + /* * Ensure an allocation request for @size will succeed by freeing enough memory. * @@ -2940,11 +3103,11 @@ sgen_ensure_free_space (size_t size) if (generation_to_collect == -1) return; - sgen_perform_collection (size, generation_to_collect, reason); + sgen_perform_collection (size, generation_to_collect, reason, generation_to_collect == GENERATION_NURSERY); } void -sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason) +sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish) { TV_DECLARE (gc_end); GGTimingInfo infos [2]; @@ -2960,7 +3123,13 @@ sgen_perform_collection (size_t requested_size, int generation_to_collect, const TV_GETTIME (infos [0].total_time); infos [1].generation = -1; - stop_world (generation_to_collect); + sgen_stop_world (generation_to_collect); + + if (concurrent_collection_in_progress) { + g_assert (generation_to_collect == GENERATION_NURSERY); + major_finish_concurrent_collection (); + } + //FIXME extract overflow reason if (generation_to_collect == GENERATION_NURSERY) { if (collect_nursery ()) { @@ -2968,9 +3137,18 @@ sgen_perform_collection (size_t requested_size, int generation_to_collect, const overflow_reason = "Minor overflow"; } } else { - if (major_do_collection (reason)) { - overflow_generation_to_collect = GENERATION_NURSERY; - overflow_reason = "Excessive pinning"; + if (major_collector.is_concurrent) + collect_nursery (); + + if (major_collector.is_concurrent && !wait_to_finish) { + major_start_concurrent_collection (reason); + // FIXME: set infos[0] properly + goto done; + } else { + if (major_do_collection (reason)) { + overflow_generation_to_collect = GENERATION_NURSERY; + overflow_reason = "Excessive pinning"; + } } } @@ -2978,7 +3156,7 @@ sgen_perform_collection (size_t requested_size, int generation_to_collect, const infos [0].total_time = SGEN_TV_ELAPSED (infos [0].total_time, gc_end); - if (overflow_generation_to_collect != -1) { + if (!major_collector.is_concurrent && overflow_generation_to_collect != -1) { mono_profiler_gc_event (MONO_GC_EVENT_START, overflow_generation_to_collect); infos [1].generation = overflow_generation_to_collect; infos [1].reason = overflow_reason; @@ -2997,17 +3175,18 @@ sgen_perform_collection (size_t requested_size, int generation_to_collect, const mono_profiler_gc_event (MONO_GC_EVENT_END, overflow_generation_to_collect); } - DEBUG (2, fprintf (gc_debug_file, "Heap size: %lu, LOS size: %lu\n", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage)); + SGEN_LOG (2, "Heap size: %lu, LOS size: %lu", (unsigned long)mono_gc_get_heap_size (), (unsigned long)los_memory_usage); /* this also sets the proper pointers for the next allocation */ if (generation_to_collect == GENERATION_NURSERY && !sgen_can_alloc_size (requested_size)) { /* TypeBuilder and MonoMethod are killing mcs with fragmentation */ - DEBUG (1, fprintf (gc_debug_file, "nursery collection didn't find enough room for %zd alloc (%d pinned)\n", requested_size, sgen_get_pinned_count ())); + SGEN_LOG (1, "nursery collection didn't find enough room for %zd alloc (%d pinned)", requested_size, sgen_get_pinned_count ()); sgen_dump_pin_queue (); degraded_mode = 1; } - restart_world (generation_to_collect, infos); + done: + sgen_restart_world (generation_to_collect, infos); mono_profiler_gc_event (MONO_GC_EVENT_END, generation_to_collect); } @@ -3048,11 +3227,22 @@ report_internal_mem_usage (void) static inline gboolean sgen_is_object_alive (void *object) { + mword objsize; + if (ptr_in_nursery (object)) return sgen_nursery_is_object_alive (object); /* Oldgen objects can be pinned and forwarded too */ if (SGEN_OBJECT_IS_PINNED (object) || SGEN_OBJECT_IS_FORWARDED (object)) return TRUE; + + /* + * FIXME: major_collector.is_object_live() also calculates the + * size. Avoid the double calculation. + */ + objsize = SGEN_ALIGN_UP (sgen_safe_object_get_size ((MonoObject*)object)); + if (objsize > SGEN_MAX_SMALL_OBJ_SIZE) + return sgen_los_object_is_pinned (object); + return major_collector.is_object_live (object); } @@ -3075,8 +3265,9 @@ has_critical_finalizer (MonoObject *obj) return mono_class_has_parent_fast (class, mono_defaults.critical_finalizer_object); } -static void -queue_finalization_entry (MonoObject *obj) { +void +sgen_queue_finalization_entry (MonoObject *obj) +{ FinalizeReadyEntry *entry = sgen_alloc_internal (INTERNAL_MEM_FINALIZE_READY_ENTRY); entry->object = obj; if (has_critical_finalizer (obj)) { @@ -3098,8 +3289,6 @@ object_is_reachable (char *object, char *start, char *end) return sgen_is_object_alive (object); } -#include "sgen-fin-weak-hash.c" - gboolean sgen_object_is_live (void *obj) { @@ -3153,7 +3342,7 @@ clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char if (!object_is_reachable (object, start, end)) { EphemeronLinkNode *tmp = current; - DEBUG (5, fprintf (gc_debug_file, "Dead Ephemeron array at %p\n", object)); + SGEN_LOG (5, "Dead Ephemeron array at %p", object); if (prev) prev->next = current->next; @@ -3173,7 +3362,7 @@ clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char /*The array was promoted, add global remsets for key/values left behind in nursery.*/ was_promoted = was_in_nursery && !ptr_in_nursery (object); - DEBUG (5, fprintf (gc_debug_file, "Clearing unreachable entries for ephemeron array at %p\n", object)); + SGEN_LOG (5, "Clearing unreachable entries for ephemeron array at %p", object); array = (MonoArray*)object; cur = mono_array_addr (array, Ephemeron, 0); @@ -3186,9 +3375,9 @@ clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char if (!key || key == tombstone) continue; - DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0), + SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0), key, object_is_reachable (key, start, end) ? "reachable" : "unreachable", - cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable")); + cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"); if (!object_is_reachable (key, start, end)) { cur->key = tombstone; @@ -3198,11 +3387,11 @@ clear_unreachable_ephemerons (CopyOrMarkObjectFunc copy_func, char *start, char if (was_promoted) { if (ptr_in_nursery (key)) {/*key was not promoted*/ - DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to key %p\n", key)); + SGEN_LOG (5, "\tAdded remset to key %p", key); sgen_add_to_global_remset (&cur->key); } if (ptr_in_nursery (cur->value)) {/*value was not promoted*/ - DEBUG (5, fprintf (gc_debug_file, "\tAdded remset to value %p\n", cur->value)); + SGEN_LOG (5, "\tAdded remset to value %p", cur->value); sgen_add_to_global_remset (&cur->value); } } @@ -3224,7 +3413,7 @@ mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end for (current = ephemeron_list; current; current = current->next) { char *object = current->array; - DEBUG (5, fprintf (gc_debug_file, "Ephemeron array at %p\n", object)); + SGEN_LOG (5, "Ephemeron array at %p", object); /* For now we process all ephemerons during all collections. @@ -3238,7 +3427,7 @@ mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end /*It has to be alive*/ if (!object_is_reachable (object, start, end)) { - DEBUG (5, fprintf (gc_debug_file, "\tnot reachable\n")); + SGEN_LOG (5, "\tnot reachable"); continue; } @@ -3255,9 +3444,9 @@ mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end if (!key || key == tombstone) continue; - DEBUG (5, fprintf (gc_debug_file, "[%td] key %p (%s) value %p (%s)\n", cur - mono_array_addr (array, Ephemeron, 0), + SGEN_LOG (5, "[%td] key %p (%s) value %p (%s)", cur - mono_array_addr (array, Ephemeron, 0), key, object_is_reachable (key, start, end) ? "reachable" : "unreachable", - cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable")); + cur->value, cur->value && object_is_reachable (cur->value, start, end) ? "reachable" : "unreachable"); if (object_is_reachable (key, start, end)) { char *value = cur->value; @@ -3272,7 +3461,7 @@ mark_ephemerons_in_range (CopyOrMarkObjectFunc copy_func, char *start, char *end } } - DEBUG (5, fprintf (gc_debug_file, "Ephemeron run finished. Is it done %d\n", nothing_marked)); + SGEN_LOG (5, "Ephemeron run finished. Is it done %d", nothing_marked); return nothing_marked; } @@ -3321,7 +3510,7 @@ mono_gc_invoke_finalizers (void) num_ready_finalizers--; obj = entry->object; entry->object = NULL; - DEBUG (7, fprintf (gc_debug_file, "Finalizing object %p (%s)\n", obj, safe_name (obj))); + SGEN_LOG (7, "Finalizing object %p (%s)", obj, safe_name (obj)); } UNLOCK_GC; @@ -3382,7 +3571,7 @@ mono_gc_register_root_inner (char *start, size_t size, void *descr, int root_typ sgen_hash_table_replace (&roots_hash [root_type], start, &new_root, NULL); roots_size += size; - DEBUG (3, fprintf (gc_debug_file, "Added root for range: %p-%p, descr: %p (%d/%d bytes)\n", start, new_root.end_root, descr, (int)size, (int)roots_size)); + SGEN_LOG (3, "Added root for range: %p-%p, descr: %p (%d/%d bytes)", start, new_root.end_root, descr, (int)size, (int)roots_size); UNLOCK_GC; return TRUE; @@ -3422,36 +3611,6 @@ mono_gc_deregister_root (char* addr) unsigned int sgen_global_stop_count = 0; -#ifdef USE_MONO_CTX -static MonoContext cur_thread_ctx = {0}; -#else -static mword cur_thread_regs [ARCH_NUM_REGS] = {0}; -#endif - -static void -update_current_thread_stack (void *start) -{ - int stack_guard = 0; -#ifndef USE_MONO_CTX - void *ptr = cur_thread_regs; -#endif - SgenThreadInfo *info = mono_thread_info_current (); - - info->stack_start = align_pointer (&stack_guard); - g_assert (info->stack_start >= info->stack_start_limit && info->stack_start < info->stack_end); -#ifdef USE_MONO_CTX - MONO_CONTEXT_GET_CURRENT (cur_thread_ctx); - info->monoctx = &cur_thread_ctx; - if (gc_callbacks.thread_suspend_func) - gc_callbacks.thread_suspend_func (info->runtime_data, NULL, info->monoctx); -#else - ARCH_STORE_REGS (ptr); - info->stopped_regs = ptr; - if (gc_callbacks.thread_suspend_func) - gc_callbacks.thread_suspend_func (info->runtime_data, NULL, NULL); -#endif -} - void sgen_fill_thread_info_for_suspend (SgenThreadInfo *info) { @@ -3459,185 +3618,6 @@ sgen_fill_thread_info_for_suspend (SgenThreadInfo *info) remset.fill_thread_info_for_suspend (info); } -static gboolean -is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip); - -static int -restart_threads_until_none_in_managed_allocator (void) -{ - SgenThreadInfo *info; - int num_threads_died = 0; - int sleep_duration = -1; - - for (;;) { - int restart_count = 0, restarted_count = 0; - /* restart all threads that stopped in the - allocator */ - FOREACH_THREAD_SAFE (info) { - gboolean result; - if (info->skip || info->gc_disabled || !info->joined_stw) - continue; - if (!info->thread_is_dying && (!info->stack_start || info->in_critical_region || - is_ip_in_managed_allocator (info->stopped_domain, info->stopped_ip))) { - binary_protocol_thread_restart ((gpointer)mono_thread_info_get_tid (info)); - result = sgen_resume_thread (info); - if (result) { - ++restart_count; - } else { - info->skip = 1; - } - } else { - /* we set the stopped_ip to - NULL for threads which - we're not restarting so - that we can easily identify - the others */ - info->stopped_ip = NULL; - info->stopped_domain = NULL; - } - } END_FOREACH_THREAD_SAFE - /* if no threads were restarted, we're done */ - if (restart_count == 0) - break; - - /* wait for the threads to signal their restart */ - sgen_wait_for_suspend_ack (restart_count); - - if (sleep_duration < 0) { -#ifdef HOST_WIN32 - SwitchToThread (); -#else - sched_yield (); -#endif - sleep_duration = 0; - } else { - g_usleep (sleep_duration); - sleep_duration += 10; - } - - /* stop them again */ - FOREACH_THREAD (info) { - gboolean result; - if (info->skip || info->stopped_ip == NULL) - continue; - result = sgen_suspend_thread (info); - - if (result) { - ++restarted_count; - } else { - info->skip = 1; - } - } END_FOREACH_THREAD - /* some threads might have died */ - num_threads_died += restart_count - restarted_count; - /* wait for the threads to signal their suspension - again */ - sgen_wait_for_suspend_ack (restarted_count); - } - - return num_threads_died; -} - -static void -acquire_gc_locks (void) -{ - LOCK_INTERRUPTION; - mono_thread_info_suspend_lock (); -} - -static void -release_gc_locks (void) -{ - mono_thread_info_suspend_unlock (); - UNLOCK_INTERRUPTION; -} - -static TV_DECLARE (stop_world_time); -static unsigned long max_pause_usec = 0; - -/* LOCKING: assumes the GC lock is held */ -static int -stop_world (int generation) -{ - int count, dead; - - /*XXX this is the right stop, thought might not be the nicest place to put it*/ - sgen_process_togglerefs (); - - mono_profiler_gc_event (MONO_GC_EVENT_PRE_STOP_WORLD, generation); - acquire_gc_locks (); - - update_current_thread_stack (&count); - - sgen_global_stop_count++; - DEBUG (3, fprintf (gc_debug_file, "stopping world n %d from %p %p\n", sgen_global_stop_count, mono_thread_info_current (), (gpointer)mono_native_thread_id_get ())); - TV_GETTIME (stop_world_time); - count = sgen_thread_handshake (TRUE); - dead = restart_threads_until_none_in_managed_allocator (); - if (count < dead) - g_error ("More threads have died (%d) that been initialy suspended %d", dead, count); - count -= dead; - - DEBUG (3, fprintf (gc_debug_file, "world stopped %d thread(s)\n", count)); - mono_profiler_gc_event (MONO_GC_EVENT_POST_STOP_WORLD, generation); - - sgen_memgov_collection_start (generation); - - return count; -} - -/* LOCKING: assumes the GC lock is held */ -static int -restart_world (int generation, GGTimingInfo *timing) -{ - int count; - SgenThreadInfo *info; - TV_DECLARE (end_sw); - TV_DECLARE (end_bridge); - unsigned long usec, bridge_usec; - - /* notify the profiler of the leftovers */ - if (G_UNLIKELY (mono_profiler_events & MONO_PROFILE_GC_MOVES)) { - if (moved_objects_idx) { - mono_profiler_gc_moves (moved_objects, moved_objects_idx); - moved_objects_idx = 0; - } - } - mono_profiler_gc_event (MONO_GC_EVENT_PRE_START_WORLD, generation); - FOREACH_THREAD (info) { - info->stack_start = NULL; -#ifdef USE_MONO_CTX - info->monoctx = NULL; -#else - info->stopped_regs = NULL; -#endif - } END_FOREACH_THREAD - - stw_bridge_process (); - release_gc_locks (); - - count = sgen_thread_handshake (FALSE); - TV_GETTIME (end_sw); - usec = TV_ELAPSED (stop_world_time, end_sw); - max_pause_usec = MAX (usec, max_pause_usec); - DEBUG (2, fprintf (gc_debug_file, "restarted %d thread(s) (pause time: %d usec, max: %d)\n", count, (int)usec, (int)max_pause_usec)); - mono_profiler_gc_event (MONO_GC_EVENT_POST_START_WORLD, generation); - - bridge_process (); - - TV_GETTIME (end_bridge); - bridge_usec = TV_ELAPSED (end_sw, end_bridge); - - if (timing) { - timing [0].stw_time = usec; - timing [0].bridge_time = bridge_usec; - } - - sgen_memgov_collection_end (generation, timing, timing ? 2 : 0); - - return count; -} - int sgen_get_current_collection_generation (void) { @@ -3686,20 +3666,20 @@ scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, Gray FOREACH_THREAD (info) { if (info->skip) { - DEBUG (3, fprintf (gc_debug_file, "Skipping dead thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start)); + SGEN_LOG (3, "Skipping dead thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start); continue; } if (info->gc_disabled) { - DEBUG (3, fprintf (gc_debug_file, "GC disabled for thread %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start)); + SGEN_LOG (3, "GC disabled for thread %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start); continue; } if (!info->joined_stw) { - DEBUG (3, fprintf (gc_debug_file, "Skipping thread not seen in STW %p, range: %p-%p, size: %td\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start)); + SGEN_LOG (3, "Skipping thread not seen in STW %p, range: %p-%p, size: %td", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start); continue; } - DEBUG (3, fprintf (gc_debug_file, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d\n", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ())); + SGEN_LOG (3, "Scanning thread %p, range: %p-%p, size: %td, pinned=%d", info, info->stack_start, info->stack_end, (char*)info->stack_end - (char*)info->stack_start, sgen_get_pinned_count ()); if (!info->thread_is_dying) { if (gc_callbacks.thread_mark_func && !conservative_stack_mark) { UserCopyOrMarkData data = { NULL, queue }; @@ -3711,47 +3691,16 @@ scan_thread_data (void *start_nursery, void *end_nursery, gboolean precise, Gray } } + if (!info->thread_is_dying && !precise) { #ifdef USE_MONO_CTX - if (!info->thread_is_dying && !precise) - conservatively_pin_objects_from ((void**)info->monoctx, (void**)info->monoctx + ARCH_NUM_REGS, + conservatively_pin_objects_from ((void**)&info->ctx, (void**)&info->ctx + ARCH_NUM_REGS, start_nursery, end_nursery, PIN_TYPE_STACK); #else - if (!info->thread_is_dying && !precise) - conservatively_pin_objects_from (info->stopped_regs, info->stopped_regs + ARCH_NUM_REGS, + conservatively_pin_objects_from (&info->regs, &info->regs + ARCH_NUM_REGS, start_nursery, end_nursery, PIN_TYPE_STACK); #endif - } END_FOREACH_THREAD -} - -static void -find_pinning_ref_from_thread (char *obj, size_t size) -{ - int j; - SgenThreadInfo *info; - char *endobj = obj + size; - - FOREACH_THREAD (info) { - char **start = (char**)info->stack_start; - if (info->skip) - continue; - while (start < (char**)info->stack_end) { - if (*start >= obj && *start < endobj) { - DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in thread %p (id %p) at %p, stack: %p-%p\n", obj, info, (gpointer)mono_thread_info_get_tid (info), start, info->stack_start, info->stack_end)); - } - start++; } - - for (j = 0; j < ARCH_NUM_REGS; ++j) { -#ifdef USE_MONO_CTX - mword w = ((mword*)info->monoctx) [j]; -#else - mword w = (mword)info->stopped_regs [j]; -#endif - - if (w >= (mword)obj && w < (mword)obj + size) - DEBUG (0, fprintf (gc_debug_file, "Object %p referenced in saved reg %d of thread %p (id %p)\n", obj, j, info, (gpointer)mono_thread_info_get_tid (info))); - } END_FOREACH_THREAD - } + } END_FOREACH_THREAD } static gboolean @@ -3779,7 +3728,7 @@ sgen_thread_register (SgenThreadInfo* info, void *addr) g_assert (!mono_native_tls_get_value (thread_info_key)); mono_native_tls_set_value (thread_info_key, info); #else - thread_info = info; + sgen_thread_info = info; #endif #if !defined(__MACH__) @@ -3796,9 +3745,9 @@ sgen_thread_register (SgenThreadInfo* info, void *addr) info->stopped_ip = NULL; info->stopped_domain = NULL; #ifdef USE_MONO_CTX - info->monoctx = NULL; + memset (&info->ctx, 0, sizeof (MonoContext)); #else - info->stopped_regs = NULL; + memset (&info->regs, 0, sizeof (info->regs)); #endif sgen_init_tlab_info (info); @@ -3809,10 +3758,6 @@ sgen_thread_register (SgenThreadInfo* info, void *addr) store_remset_buffer_index_addr = &store_remset_buffer_index; #endif -#if defined(__MACH__) - info->mach_port = mach_thread_self (); -#endif - /* try to get it with attributes first */ #if defined(HAVE_PTHREAD_GETATTR_NP) && defined(HAVE_PTHREAD_ATTR_GETSTACK) { @@ -3845,7 +3790,7 @@ sgen_thread_register (SgenThreadInfo* info, void *addr) if (remset.register_thread) remset.register_thread (info); - DEBUG (3, fprintf (gc_debug_file, "registered thread %p (%p) stack end %p\n", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end)); + SGEN_LOG (3, "registered thread %p (%p) stack end %p", info, (gpointer)mono_thread_info_get_tid (info), info->stack_end); if (gc_callbacks.thread_attach_func) info->runtime_data = gc_callbacks.thread_attach_func (); @@ -3906,11 +3851,7 @@ sgen_thread_unregister (SgenThreadInfo *p) #endif binary_protocol_thread_unregister ((gpointer)mono_thread_info_get_tid (p)); - DEBUG (3, fprintf (gc_debug_file, "unregister thread %p (%p)\n", p, (gpointer)mono_thread_info_get_tid (p))); - -#if defined(__MACH__) - mach_port_deallocate (current_task (), p->mach_port); -#endif + SGEN_LOG (3, "unregister thread %p (%p)", p, (gpointer)mono_thread_info_get_tid (p)); if (gc_callbacks.thread_detach_func) { gc_callbacks.thread_detach_func (p->runtime_data); @@ -4009,7 +3950,7 @@ mono_gc_wbarrier_set_field (MonoObject *obj, gpointer field_ptr, MonoObject* val *(void**)field_ptr = value; return; } - DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", field_ptr)); + SGEN_LOG (8, "Adding remset at %p", field_ptr); if (value) binary_protocol_wbarrier (field_ptr, value, value->vtable); @@ -4024,7 +3965,7 @@ mono_gc_wbarrier_set_arrayref (MonoArray *arr, gpointer slot_ptr, MonoObject* va *(void**)slot_ptr = value; return; } - DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", slot_ptr)); + SGEN_LOG (8, "Adding remset at %p", slot_ptr); if (value) binary_protocol_wbarrier (slot_ptr, value, value->vtable); @@ -4099,6 +4040,8 @@ find_object_for_ptr (char *ptr) void mono_gc_wbarrier_generic_nostore (gpointer ptr) { + gpointer obj; + HEAVY_STAT (++stat_wbarrier_generic_store); #ifdef XDOMAIN_CHECKS_IN_WBARRIER @@ -4117,15 +4060,25 @@ mono_gc_wbarrier_generic_nostore (gpointer ptr) } #endif - if (*(gpointer*)ptr) - binary_protocol_wbarrier (ptr, *(gpointer*)ptr, (gpointer)LOAD_VTABLE (*(gpointer*)ptr)); + obj = *(gpointer*)ptr; + if (obj) + binary_protocol_wbarrier (ptr, obj, (gpointer)LOAD_VTABLE (obj)); - if (ptr_in_nursery (ptr) || ptr_on_stack (ptr) || !ptr_in_nursery (*(gpointer*)ptr)) { - DEBUG (8, fprintf (gc_debug_file, "Skipping remset at %p\n", ptr)); + if (ptr_in_nursery (ptr) || ptr_on_stack (ptr)) { + SGEN_LOG (8, "Skipping remset at %p", ptr); return; } - DEBUG (8, fprintf (gc_debug_file, "Adding remset at %p\n", ptr)); + /* + * We need to record old->old pointer locations for the + * concurrent collector. + */ + if (!ptr_in_nursery (obj) && !concurrent_collection_in_progress) { + SGEN_LOG (8, "Skipping remset at %p", ptr); + return; + } + + SGEN_LOG (8, "Adding remset at %p", ptr); remset.wbarrier_generic_nostore (ptr); } @@ -4133,7 +4086,7 @@ mono_gc_wbarrier_generic_nostore (gpointer ptr) void mono_gc_wbarrier_generic_store (gpointer ptr, MonoObject* value) { - DEBUG (8, fprintf (gc_debug_file, "Wbarrier store at %p to %p (%s)\n", ptr, value, value ? safe_name (value) : "null")); + SGEN_LOG (8, "Wbarrier store at %p to %p (%s)", ptr, value, value ? safe_name (value) : "null"); *(void**)ptr = value; if (ptr_in_nursery (value)) mono_gc_wbarrier_generic_nostore (ptr); @@ -4181,7 +4134,7 @@ mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass * HEAVY_STAT (++stat_wbarrier_value_copy); g_assert (klass->valuetype); - DEBUG (8, fprintf (gc_debug_file, "Adding value remset at %p, count %d, descr %p for class %s (%p)\n", dest, count, klass->gc_descr, klass->name, klass)); + SGEN_LOG (8, "Adding value remset at %p, count %d, descr %p for class %s (%p)", dest, count, klass->gc_descr, klass->name, klass); if (ptr_in_nursery (dest) || ptr_on_stack (dest) || !SGEN_CLASS_HAS_REFERENCES (klass)) { size_t element_size = mono_class_value_size (klass, NULL); @@ -4192,6 +4145,7 @@ mono_gc_wbarrier_value_copy (gpointer dest, gpointer src, int count, MonoClass * #ifdef SGEN_BINARY_PROTOCOL { + size_t element_size = mono_class_value_size (klass, NULL); int i; for (i = 0; i < count; ++i) { scan_object_for_binary_protocol_copy_wbarrier ((char*)dest + i * element_size, @@ -4321,7 +4275,7 @@ mono_gc_collect (int generation) LOCK_GC; if (generation > 1) generation = 1; - sgen_perform_collection (0, generation, "user request"); + sgen_perform_collection (0, generation, "user request", TRUE); UNLOCK_GC; } @@ -4402,21 +4356,39 @@ mono_gc_enable_events (void) void mono_gc_weak_link_add (void **link_addr, MonoObject *obj, gboolean track) { - mono_gc_register_disappearing_link (obj, link_addr, track, FALSE); + sgen_register_disappearing_link (obj, link_addr, track, FALSE); } void mono_gc_weak_link_remove (void **link_addr) { - mono_gc_register_disappearing_link (NULL, link_addr, FALSE, FALSE); + sgen_register_disappearing_link (NULL, link_addr, FALSE, FALSE); } MonoObject* mono_gc_weak_link_get (void **link_addr) { - if (!*link_addr) + /* + * We must only load *link_addr once because it might change + * under our feet, and REVEAL_POINTER (NULL) results in an + * invalid reference. + */ + void *ptr = *link_addr; + if (!ptr) return NULL; - return (MonoObject*) REVEAL_POINTER (*link_addr); + + /* + * During the second bridge processing step the world is + * running again. That step processes all weak links once + * more to null those that refer to dead objects. Before that + * is completed, those links must not be followed, so we + * conservatively wait for bridge processing when any weak + * link is dereferenced. + */ + if (G_UNLIKELY (bridge_processing_in_progress)) + mono_gc_wait_for_bridge_processing (); + + return (MonoObject*) REVEAL_POINTER (ptr); } gboolean @@ -4435,7 +4407,7 @@ mono_gc_ephemeron_array_add (MonoObject *obj) node->next = ephemeron_list; ephemeron_list = node; - DEBUG (5, fprintf (gc_debug_file, "Registered ephemeron array %p\n", obj)); + SGEN_LOG (5, "Registered ephemeron array %p", obj); UNLOCK_GC; return TRUE; @@ -4464,7 +4436,7 @@ mono_gc_is_gc_thread (void) static gboolean is_critical_method (MonoMethod *method) { - return mono_runtime_is_critical_method (method) || mono_gc_is_critical_method (method); + return mono_runtime_is_critical_method (method) || sgen_is_critical_method (method); } void @@ -4516,7 +4488,7 @@ mono_gc_base_init (void) mono_threads_init (&cb, sizeof (SgenThreadInfo)); - LOCK_INIT (interruption_mutex); + LOCK_INIT (sgen_interruption_mutex); LOCK_INIT (pin_queue_mutex); init_user_copy_or_mark_key (); @@ -4582,6 +4554,8 @@ mono_gc_base_init (void) sgen_marksweep_par_init (&major_collector); } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-fixed-par")) { sgen_marksweep_fixed_par_init (&major_collector); + } else if (!major_collector_opt || !strcmp (major_collector_opt, "marksweep-conc")) { + sgen_marksweep_conc_init (&major_collector); } else if (!strcmp (major_collector_opt, "copying")) { sgen_copying_init (&major_collector); } else { @@ -4616,6 +4590,10 @@ mono_gc_base_init (void) if (g_str_has_prefix (opt, "wbarrier=")) { opt = strchr (opt, '=') + 1; if (strcmp (opt, "remset") == 0) { + if (major_collector.is_concurrent) { + fprintf (stderr, "The concurrent collector does not support the SSB write barrier.\n"); + exit (1); + } use_cardtable = FALSE; } else if (strcmp (opt, "cardtable") == 0) { if (!use_cardtable) { @@ -4762,7 +4740,7 @@ mono_gc_base_init (void) fprintf (stderr, " max-heap-size=N (where N is an integer, possibly with a k, m or a g suffix)\n"); fprintf (stderr, " soft-heap-limit=n (where N is an integer, possibly with a k, m or a g suffix)\n"); fprintf (stderr, " nursery-size=N (where N is an integer, possibly with a k, m or a g suffix)\n"); - fprintf (stderr, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-par' or `copying')\n"); + fprintf (stderr, " major=COLLECTOR (where COLLECTOR is `marksweep', `marksweep-par', 'marksweep-fixed', 'marksweep-fixed-par' or `copying')\n"); fprintf (stderr, " minor=COLLECTOR (where COLLECTOR is `simple' or `split')\n"); fprintf (stderr, " wbarrier=WBARRIER (where WBARRIER is `remset' or `cardtable')\n"); fprintf (stderr, " stack-mark=MARK-METHOD (where MARK-METHOD is 'precise' or 'conservative')\n"); @@ -4780,6 +4758,8 @@ mono_gc_base_init (void) if (major_collector.is_parallel) sgen_workers_init (num_workers); + else if (major_collector.is_concurrent) + sgen_workers_init (1); if (major_collector_opt) g_free (major_collector_opt); @@ -4787,6 +4767,9 @@ mono_gc_base_init (void) if (minor_collector_opt) g_free (minor_collector_opt); + if (major_collector.is_concurrent) + LOCK_INIT (workers_distribute_gray_queue_mutex); + alloc_nursery (); if ((env = getenv ("MONO_GC_DEBUG"))) { @@ -4844,6 +4827,8 @@ mono_gc_base_init (void) do_verify_nursery = TRUE; } else if (!strcmp (opt, "dump-nursery-at-minor-gc")) { do_dump_nursery_content = TRUE; + } else if (!strcmp (opt, "no-managed-allocator")) { + sgen_set_use_managed_allocator (FALSE); } else if (!strcmp (opt, "disable-minor")) { disable_minor_collections = TRUE; } else if (!strcmp (opt, "disable-major")) { @@ -4871,12 +4856,21 @@ mono_gc_base_init (void) fprintf (stderr, " verify-before-allocs[=]\n"); fprintf (stderr, " check-at-minor-collections\n"); fprintf (stderr, " verify-before-collections\n"); + fprintf (stderr, " verify-nursery-at-minor-gc\n"); + fprintf (stderr, " dump-nursery-at-minor-gc\n"); fprintf (stderr, " disable-minor\n"); fprintf (stderr, " disable-major\n"); fprintf (stderr, " xdomain-checks\n"); fprintf (stderr, " clear-at-gc\n"); + fprintf (stderr, " clear-nursery-at-gc\n"); + fprintf (stderr, " check-scan-starts\n"); + fprintf (stderr, " no-managed-allocator\n"); fprintf (stderr, " print-allowance\n"); fprintf (stderr, " print-pinning\n"); + fprintf (stderr, " heap-dump=\n"); +#ifdef SGEN_BINARY_PROTOCOL + fprintf (stderr, " binary-protocol=\n"); +#endif exit (1); } } @@ -4922,28 +4916,16 @@ mono_gc_get_gc_name (void) static MonoMethod *write_barrier_method; -static gboolean -mono_gc_is_critical_method (MonoMethod *method) +gboolean +sgen_is_critical_method (MonoMethod *method) { return (method == write_barrier_method || sgen_is_managed_allocator (method)); } -static gboolean -is_ip_in_managed_allocator (MonoDomain *domain, gpointer ip) +gboolean +sgen_has_critical_method (void) { - MonoJitInfo *ji; - - if (!mono_thread_internal_current ()) - /* Happens during thread attach */ - return FALSE; - - if (!ip || !domain) - return FALSE; - ji = mono_jit_info_table_find (domain, ip); - if (!ji) - return FALSE; - - return mono_gc_is_critical_method (ji->method); + return write_barrier_method || sgen_has_managed_allocator (); } static void @@ -4962,13 +4944,15 @@ emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels) mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS); nursery_check_return_labels [0] = mono_mb_emit_branch (mb, CEE_BEQ); - // if (!ptr_in_nursery (*ptr)) return; - mono_mb_emit_ldarg (mb, 0); - mono_mb_emit_byte (mb, CEE_LDIND_I); - mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS); - mono_mb_emit_byte (mb, CEE_SHR_UN); - mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS); - nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN); + if (!major_collector.is_concurrent) { + // if (!ptr_in_nursery (*ptr)) return; + mono_mb_emit_ldarg (mb, 0); + mono_mb_emit_byte (mb, CEE_LDIND_I); + mono_mb_emit_icon (mb, DEFAULT_NURSERY_BITS); + mono_mb_emit_byte (mb, CEE_SHR_UN); + mono_mb_emit_icon (mb, (mword)sgen_get_nursery_start () >> DEFAULT_NURSERY_BITS); + nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BNE_UN); + } #else int label_continue1, label_continue2; int dereferenced_var; @@ -4996,15 +4980,17 @@ emit_nursery_check (MonoMethodBuilder *mb, int *nursery_check_return_labels) mono_mb_emit_byte (mb, CEE_LDIND_I); mono_mb_emit_stloc (mb, dereferenced_var); - // if (*ptr < sgen_get_nursery_start ()) return; - mono_mb_emit_ldloc (mb, dereferenced_var); - mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ()); - nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT); + if (!major_collector.is_concurrent) { + // if (*ptr < sgen_get_nursery_start ()) return; + mono_mb_emit_ldloc (mb, dereferenced_var); + mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_start ()); + nursery_check_return_labels [1] = mono_mb_emit_branch (mb, CEE_BLT); - // if (*ptr >= sgen_get_nursery_end ()) return; - mono_mb_emit_ldloc (mb, dereferenced_var); - mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ()); - nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE); + // if (*ptr >= sgen_get_nursery_end ()) return; + mono_mb_emit_ldloc (mb, dereferenced_var); + mono_mb_emit_ptr (mb, (gpointer) sgen_get_nursery_end ()); + nursery_check_return_labels [2] = mono_mb_emit_branch (mb, CEE_BGE); + } #endif } @@ -5213,25 +5199,6 @@ mono_gc_is_disabled (void) return FALSE; } -void -sgen_debug_printf (int level, const char *format, ...) -{ - va_list ap; - - if (level > gc_debug_level) - return; - - va_start (ap, format); - vfprintf (gc_debug_file, format, ap); - va_end (ap); -} - -FILE* -sgen_get_logfile (void) -{ - return gc_debug_file; -} - #ifdef HOST_WIN32 BOOL APIENTRY mono_gc_dllmain (HMODULE module_handle, DWORD reason, LPVOID reserved) { @@ -5293,7 +5260,7 @@ sgen_major_collector_iterate_live_block_ranges (sgen_cardtable_block_callback ca void sgen_major_collector_scan_card_table (SgenGrayQueue *queue) { - major_collector.scan_card_table (queue); + major_collector.scan_card_table (FALSE, queue); } SgenMajorCollector* @@ -5335,10 +5302,19 @@ mono_gc_register_altstack (gpointer stack, gint32 stack_size, gpointer altstack, void sgen_check_whole_heap_stw (void) { - stop_world (0); + sgen_stop_world (0); sgen_clear_nursery_fragments (); sgen_check_whole_heap (); - restart_world (0, NULL); + sgen_restart_world (0, NULL); +} + +void +sgen_gc_event_moves (void) +{ + if (moved_objects_idx) { + mono_profiler_gc_moves (moved_objects, moved_objects_idx); + moved_objects_idx = 0; + } } #endif /* HAVE_SGEN_GC */