X-Git-Url: http://wien.tomnetworks.com/gitweb/?a=blobdiff_plain;f=mono%2Fsgen%2Fsgen-gc.c;h=37a162b240c3c44a9957d69a072ea18a6ff20301;hb=52d58c1c5d1770f2c45cbe22b6f2a80b1f886dcc;hp=1956af35772abe1839f3fc8736377b71161fbf87;hpb=0b7588ee2e8732c22dce914662d36be388def54d;p=mono.git diff --git a/mono/sgen/sgen-gc.c b/mono/sgen/sgen-gc.c index 1956af35772..37a162b240c 100644 --- a/mono/sgen/sgen-gc.c +++ b/mono/sgen/sgen-gc.c @@ -18,18 +18,7 @@ * Copyright 2011 Xamarin, Inc. * Copyright (C) 2012 Xamarin Inc * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License 2.0 as published by the Free Software Foundation; - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License 2.0 along with this library; if not, write to the Free - * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Licensed under the MIT license. See LICENSE file in the project root for full license information. * * Important: allocation provides always zeroed memory, having to do * a memset after allocation is deadly for performance. @@ -245,6 +234,8 @@ static gboolean do_verify_nursery = FALSE; static gboolean do_dump_nursery_content = FALSE; static gboolean enable_nursery_canaries = FALSE; +static gboolean precleaning_enabled = TRUE; + #ifdef HEAVY_STATISTICS guint64 stat_objects_alloced_degraded = 0; guint64 stat_bytes_alloced_degraded = 0; @@ -341,7 +332,6 @@ nursery_canaries_enabled (void) * ###################################################################### */ MonoCoopMutex gc_mutex; -gboolean sgen_try_free_some_memory; #define SCAN_START_SIZE SGEN_SCAN_START_SIZE @@ -706,6 +696,8 @@ pin_objects_from_nursery_pin_queue (gboolean do_scan_objects, ScanCopyContext ct definitely_pinned [count] = obj_to_pin; count++; } + if (concurrent_collection_in_progress) + sgen_pinning_register_pinned_in_nursery (obj_to_pin); next_pin_queue_entry: last = addr; @@ -1084,6 +1076,12 @@ finish_gray_stack (int generation, ScanCopyContext ctx) if (sgen_client_bridge_need_processing ()) sgen_client_bridge_reset_data (); + /* + * Mark all strong toggleref objects. This must be done before we walk ephemerons or finalizers + * to ensure they see the full set of live objects. + */ + sgen_client_mark_togglerefs (start_addr, end_addr, ctx); + /* * Walk the ephemeron tables marking all values with reachable keys. This must be completely done * before processing finalizable objects and non-tracking weak links to avoid finalizing/clearing @@ -1096,8 +1094,6 @@ finish_gray_stack (int generation, ScanCopyContext ctx) ++ephemeron_rounds; } while (!done_with_ephemerons); - sgen_client_mark_togglerefs (start_addr, end_addr, ctx); - if (sgen_client_bridge_need_processing ()) { /*Make sure the gray stack is empty before we process bridge objects so we get liveness right*/ sgen_drain_gray_stack (ctx); @@ -1167,7 +1163,7 @@ finish_gray_stack (int generation, ScanCopyContext ctx) sgen_client_clear_togglerefs (start_addr, end_addr, ctx); TV_GETTIME (btv); - SGEN_LOG (2, "Finalize queue handling scan for %s generation: %ld usecs %d ephemeron rounds", generation_name (generation), TV_ELAPSED (atv, btv), ephemeron_rounds); + SGEN_LOG (2, "Finalize queue handling scan for %s generation: %lld usecs %d ephemeron rounds", generation_name (generation), (long long)TV_ELAPSED (atv, btv), ephemeron_rounds); /* * handle disappearing links @@ -1392,7 +1388,7 @@ job_scan_major_mod_union_card_table (void *worker_data_untyped, SgenThreadPoolJo ScanCopyContext ctx = CONTEXT_FROM_OBJECT_OPERATIONS (job_data->ops, sgen_workers_get_job_gray_queue (worker_data)); g_assert (concurrent_collection_in_progress); - major_collector.scan_card_table (TRUE, ctx); + major_collector.scan_card_table (CARDTABLE_SCAN_MOD_UNION, ctx); } static void @@ -1403,7 +1399,22 @@ job_scan_los_mod_union_card_table (void *worker_data_untyped, SgenThreadPoolJob ScanCopyContext ctx = CONTEXT_FROM_OBJECT_OPERATIONS (job_data->ops, sgen_workers_get_job_gray_queue (worker_data)); g_assert (concurrent_collection_in_progress); - sgen_los_scan_card_table (TRUE, ctx); + sgen_los_scan_card_table (CARDTABLE_SCAN_MOD_UNION, ctx); +} + +static void +job_mod_union_preclean (void *worker_data_untyped, SgenThreadPoolJob *job) +{ + WorkerData *worker_data = (WorkerData *)worker_data_untyped; + ScanJob *job_data = (ScanJob*)job; + ScanCopyContext ctx = CONTEXT_FROM_OBJECT_OPERATIONS (job_data->ops, sgen_workers_get_job_gray_queue (worker_data)); + + g_assert (concurrent_collection_in_progress); + + major_collector.scan_card_table (CARDTABLE_SCAN_MOD_UNION_PRECLEAN, ctx); + sgen_los_scan_card_table (CARDTABLE_SCAN_MOD_UNION_PRECLEAN, ctx); + + sgen_scan_pin_queue_objects (ctx); } static void @@ -1463,7 +1474,7 @@ enqueue_scan_from_roots_jobs (char *heap_start, char *heap_end, SgenObjectOperat * Return whether any objects were late-pinned due to being out of memory. */ static gboolean -collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark) +collect_nursery (const char *reason, gboolean is_overflow, SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark) { gboolean needs_major; size_t max_garbage_amount; @@ -1546,7 +1557,7 @@ collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark) TV_GETTIME (atv); time_minor_pinning += TV_ELAPSED (btv, atv); - SGEN_LOG (2, "Finding pinned pointers: %zd in %ld usecs", sgen_get_pinned_count (), TV_ELAPSED (btv, atv)); + SGEN_LOG (2, "Finding pinned pointers: %zd in %lld usecs", sgen_get_pinned_count (), (long long)TV_ELAPSED (btv, atv)); SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ()); sj = (ScanJob*)sgen_thread_pool_job_alloc ("scan remset", job_remembered_set_scan, sizeof (ScanJob)); @@ -1556,12 +1567,10 @@ collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark) /* we don't have complete write barrier yet, so we scan all the old generation sections */ TV_GETTIME (btv); time_minor_scan_remsets += TV_ELAPSED (atv, btv); - SGEN_LOG (2, "Old generation scan: %ld usecs", TV_ELAPSED (atv, btv)); + SGEN_LOG (2, "Old generation scan: %lld usecs", (long long)TV_ELAPSED (atv, btv)); sgen_pin_stats_print_class_stats (); - sgen_drain_gray_stack (ctx); - /* FIXME: Why do we do this at this specific, seemingly random, point? */ sgen_client_collecting_minor (&fin_ready_queue, &critical_fin_queue); @@ -1599,7 +1608,7 @@ collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark) sgen_client_binary_protocol_reclaim_end (GENERATION_NURSERY); TV_GETTIME (btv); time_minor_fragment_creation += TV_ELAPSED (atv, btv); - SGEN_LOG (2, "Fragment creation: %ld usecs, %lu bytes available", TV_ELAPSED (atv, btv), (unsigned long)fragment_total); + SGEN_LOG (2, "Fragment creation: %lld usecs, %lu bytes available", (long long)TV_ELAPSED (atv, btv), (unsigned long)fragment_total); if (consistency_check_at_minor_collection) sgen_check_major_refs (); @@ -1629,7 +1638,7 @@ collect_nursery (SgenGrayQueue *unpin_queue, gboolean finish_up_concurrent_mark) binary_protocol_flush_buffers (FALSE); - sgen_memgov_minor_collection_end (); + sgen_memgov_minor_collection_end (reason, is_overflow); /*objects are late pinned because of lack of memory, so a major is a good call*/ needs_major = objects_pinned > 0; @@ -1700,7 +1709,7 @@ major_copy_or_mark_from_roots (size_t *old_next_pin_slot, CopyOrMarkFromRootsMod sgen_client_pre_collection_checks (); - if (!concurrent) { + if (mode != COPY_OR_MARK_FROM_ROOTS_START_CONCURRENT) { /* Remsets are not useful for a major collection */ remset.clear_cards (); } @@ -1711,8 +1720,20 @@ major_copy_or_mark_from_roots (size_t *old_next_pin_slot, CopyOrMarkFromRootsMod sgen_init_pinning (); SGEN_LOG (6, "Collecting pinned addresses"); pin_from_roots ((void*)lowest_heap_address, (void*)highest_heap_address, ctx); - + if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) { + /* Pin cemented objects that were forced */ + sgen_pin_cemented_objects (); + } sgen_optimize_pin_queue (); + if (mode == COPY_OR_MARK_FROM_ROOTS_START_CONCURRENT) { + /* + * Cemented objects that are in the pinned list will be marked. When + * marking concurrently we won't mark mod-union cards for these objects. + * Instead they will remain cemented until the next major collection, + * when we will recheck if they are still pinned in the roots. + */ + sgen_cement_force_pinned (); + } sgen_client_collecting_major_1 (); @@ -1764,24 +1785,19 @@ major_copy_or_mark_from_roots (size_t *old_next_pin_slot, CopyOrMarkFromRootsMod TV_GETTIME (btv); time_major_pinning += TV_ELAPSED (atv, btv); - SGEN_LOG (2, "Finding pinned pointers: %zd in %ld usecs", sgen_get_pinned_count (), TV_ELAPSED (atv, btv)); + SGEN_LOG (2, "Finding pinned pointers: %zd in %lld usecs", sgen_get_pinned_count (), (long long)TV_ELAPSED (atv, btv)); SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ()); major_collector.init_to_space (); - /* - * The concurrent collector doesn't move objects, neither on - * the major heap nor in the nursery, so we can mark even - * before pinning has finished. For the non-concurrent - * collector we start the workers after pinning. - */ - if (mode == COPY_OR_MARK_FROM_ROOTS_START_CONCURRENT) { - SGEN_ASSERT (0, sgen_workers_all_done (), "Why are the workers not done when we start or finish a major collection?"); - sgen_workers_start_all_workers (object_ops); - gray_queue_enable_redirect (WORKERS_DISTRIBUTE_GRAY_QUEUE); - } else if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) { + SGEN_ASSERT (0, sgen_workers_all_done (), "Why are the workers not done when we start or finish a major collection?"); + if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) { if (sgen_workers_have_idle_work ()) { - sgen_workers_start_all_workers (object_ops); + /* + * We force the finish of the worker with the new object ops context + * which can also do copying. We need to have finished pinning. + */ + sgen_workers_start_all_workers (object_ops, NULL); sgen_workers_join (); } } @@ -1797,15 +1813,29 @@ major_copy_or_mark_from_roots (size_t *old_next_pin_slot, CopyOrMarkFromRootsMod sgen_client_collecting_major_3 (&fin_ready_queue, &critical_fin_queue); - /* - * FIXME: is this the right context? It doesn't seem to contain a copy function - * unless we're concurrent. - */ - enqueue_scan_from_roots_jobs (heap_start, heap_end, object_ops, mode == COPY_OR_MARK_FROM_ROOTS_START_CONCURRENT); + enqueue_scan_from_roots_jobs (heap_start, heap_end, object_ops, FALSE); TV_GETTIME (btv); time_major_scan_roots += TV_ELAPSED (atv, btv); + /* + * We start the concurrent worker after pinning and after we scanned the roots + * in order to make sure that the worker does not finish before handling all + * the roots. + */ + if (mode == COPY_OR_MARK_FROM_ROOTS_START_CONCURRENT) { + if (precleaning_enabled) { + ScanJob *sj; + /* Mod union preclean job */ + sj = (ScanJob*)sgen_thread_pool_job_alloc ("preclean mod union cardtable", job_mod_union_preclean, sizeof (ScanJob)); + sj->ops = object_ops; + sgen_workers_start_all_workers (object_ops, &sj->job); + } else { + sgen_workers_start_all_workers (object_ops, NULL); + } + gray_queue_enable_redirect (WORKERS_DISTRIBUTE_GRAY_QUEUE); + } + if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) { ScanJob *sj; @@ -1829,11 +1859,6 @@ static void major_finish_copy_or_mark (CopyOrMarkFromRootsMode mode) { if (mode == COPY_OR_MARK_FROM_ROOTS_START_CONCURRENT) { - /* - * Prepare the pin queue for the next collection. Since pinning runs on the worker - * threads we must wait for the jobs to finish before we can reset it. - */ - sgen_workers_wait_for_jobs_finished (); sgen_finish_pinning (); sgen_pin_stats_reset (); @@ -1844,7 +1869,7 @@ major_finish_copy_or_mark (CopyOrMarkFromRootsMode mode) } static void -major_start_collection (gboolean concurrent, size_t *old_next_pin_slot) +major_start_collection (const char *reason, gboolean concurrent, size_t *old_next_pin_slot) { SgenObjectOperations *object_ops; @@ -1868,7 +1893,7 @@ major_start_collection (gboolean concurrent, size_t *old_next_pin_slot) reset_pinned_from_failed_allocation (); - sgen_memgov_major_collection_start (); + sgen_memgov_major_collection_start (concurrent, reason); //count_ref_nonref_objs (); //consistency_check (); @@ -1887,7 +1912,7 @@ major_start_collection (gboolean concurrent, size_t *old_next_pin_slot) } static void -major_finish_collection (const char *reason, size_t old_next_pin_slot, gboolean forced) +major_finish_collection (const char *reason, gboolean is_overflow, size_t old_next_pin_slot, gboolean forced) { ScannedObjectCounts counts; SgenObjectOperations *object_ops; @@ -2000,7 +2025,7 @@ major_finish_collection (const char *reason, size_t old_next_pin_slot, gboolean g_assert (sgen_gray_object_queue_is_empty (&gray_queue)); - sgen_memgov_major_collection_end (forced); + sgen_memgov_major_collection_end (forced, concurrent_collection_in_progress, reason, is_overflow); current_collection_generation = -1; memset (&counts, 0, sizeof (ScannedObjectCounts)); @@ -2022,7 +2047,7 @@ major_finish_collection (const char *reason, size_t old_next_pin_slot, gboolean } static gboolean -major_do_collection (const char *reason, gboolean forced) +major_do_collection (const char *reason, gboolean is_overflow, gboolean forced) { TV_DECLARE (time_start); TV_DECLARE (time_end); @@ -2039,8 +2064,8 @@ major_do_collection (const char *reason, gboolean forced) /* world must be stopped already */ TV_GETTIME (time_start); - major_start_collection (FALSE, &old_next_pin_slot); - major_finish_collection (reason, old_next_pin_slot, forced); + major_start_collection (reason, FALSE, &old_next_pin_slot); + major_finish_collection (reason, is_overflow, old_next_pin_slot, forced); TV_GETTIME (time_end); gc_stats.major_gc_time += TV_ELAPSED (time_start, time_end); @@ -2071,7 +2096,7 @@ major_start_concurrent_collection (const char *reason) binary_protocol_concurrent_start (); // FIXME: store reason and pass it when finishing - major_start_collection (TRUE, NULL); + major_start_collection (reason, TRUE, NULL); gray_queue_redirect (&gray_queue); @@ -2138,7 +2163,7 @@ major_finish_concurrent_collection (gboolean forced) current_collection_generation = GENERATION_OLD; sgen_cement_reset (); - major_finish_collection ("finishing", -1, forced); + major_finish_collection ("finishing", FALSE, -1, forced); if (whole_heap_check_before_collection) sgen_check_whole_heap (FALSE); @@ -2198,100 +2223,58 @@ sgen_ensure_free_space (size_t size, int generation) void sgen_perform_collection (size_t requested_size, int generation_to_collect, const char *reason, gboolean wait_to_finish) { - TV_DECLARE (gc_start); - TV_DECLARE (gc_end); TV_DECLARE (gc_total_start); TV_DECLARE (gc_total_end); - GGTimingInfo infos [2]; int overflow_generation_to_collect = -1; int oldest_generation_collected = generation_to_collect; const char *overflow_reason = NULL; + gboolean finish_concurrent = concurrent_collection_in_progress && (major_should_finish_concurrent_collection () || generation_to_collect == GENERATION_OLD); binary_protocol_collection_requested (generation_to_collect, requested_size, wait_to_finish ? 1 : 0); SGEN_ASSERT (0, generation_to_collect == GENERATION_NURSERY || generation_to_collect == GENERATION_OLD, "What generation is this?"); - TV_GETTIME (gc_start); - sgen_stop_world (generation_to_collect); TV_GETTIME (gc_total_start); - if (concurrent_collection_in_progress) { - /* - * If the concurrent worker is finished or we are asked to do a major collection - * then we finish the concurrent collection. - */ - gboolean finish = major_should_finish_concurrent_collection () || generation_to_collect == GENERATION_OLD; - - if (finish) { - major_finish_concurrent_collection (wait_to_finish); - oldest_generation_collected = GENERATION_OLD; - } else { - SGEN_ASSERT (0, generation_to_collect == GENERATION_NURSERY, "Why aren't we finishing the concurrent collection?"); + // FIXME: extract overflow reason + // FIXME: minor overflow for concurrent case + if (generation_to_collect == GENERATION_NURSERY && !finish_concurrent) { + if (concurrent_collection_in_progress) major_update_concurrent_collection (); - collect_nursery (NULL, FALSE); - } - - goto done; - } - - SGEN_ASSERT (0, !concurrent_collection_in_progress, "Why did this not get handled above?"); - /* - * There's no concurrent collection in progress. Collect the generation we're asked - * to collect. If the major collector is concurrent and we're not forced to wait, - * start a concurrent collection. - */ - // FIXME: extract overflow reason - if (generation_to_collect == GENERATION_NURSERY) { - if (collect_nursery (NULL, FALSE)) { + if (collect_nursery (reason, FALSE, NULL, FALSE) && !concurrent_collection_in_progress) { overflow_generation_to_collect = GENERATION_OLD; overflow_reason = "Minor overflow"; } + } else if (finish_concurrent) { + major_finish_concurrent_collection (wait_to_finish); + oldest_generation_collected = GENERATION_OLD; } else { + SGEN_ASSERT (0, generation_to_collect == GENERATION_OLD, "We should have handled nursery collections above"); if (major_collector.is_concurrent && !wait_to_finish) { - collect_nursery (NULL, FALSE); + collect_nursery ("Concurrent start", FALSE, NULL, FALSE); major_start_concurrent_collection (reason); - // FIXME: set infos[0] properly - goto done; - } - - if (major_do_collection (reason, wait_to_finish)) { + oldest_generation_collected = GENERATION_NURSERY; + } else if (major_do_collection (reason, FALSE, wait_to_finish)) { overflow_generation_to_collect = GENERATION_NURSERY; overflow_reason = "Excessive pinning"; } } - TV_GETTIME (gc_end); - - memset (infos, 0, sizeof (infos)); - infos [0].generation = generation_to_collect; - infos [0].reason = reason; - infos [0].is_overflow = FALSE; - infos [1].generation = -1; - infos [0].total_time = SGEN_TV_ELAPSED (gc_start, gc_end); - - SGEN_ASSERT (0, !concurrent_collection_in_progress, "Why did this not get handled above?"); - if (overflow_generation_to_collect != -1) { + SGEN_ASSERT (0, !concurrent_collection_in_progress, "We don't yet support overflow collections with the concurrent collector"); + /* * We need to do an overflow collection, either because we ran out of memory * or the nursery is fully pinned. */ - infos [1].generation = overflow_generation_to_collect; - infos [1].reason = overflow_reason; - infos [1].is_overflow = TRUE; - gc_start = gc_end; - if (overflow_generation_to_collect == GENERATION_NURSERY) - collect_nursery (NULL, FALSE); + collect_nursery (overflow_reason, TRUE, NULL, FALSE); else - major_do_collection (overflow_reason, wait_to_finish); - - TV_GETTIME (gc_end); - infos [1].total_time = SGEN_TV_ELAPSED (gc_start, gc_end); + major_do_collection (overflow_reason, TRUE, wait_to_finish); oldest_generation_collected = MAX (oldest_generation_collected, overflow_generation_to_collect); } @@ -2306,13 +2289,12 @@ sgen_perform_collection (size_t requested_size, int generation_to_collect, const degraded_mode = 1; } - done: g_assert (sgen_gray_object_queue_is_empty (&gray_queue)); TV_GETTIME (gc_total_end); time_max = MAX (time_max, TV_ELAPSED (gc_total_start, gc_total_end)); - sgen_restart_world (oldest_generation_collected, infos); + sgen_restart_world (oldest_generation_collected); } /* @@ -2933,6 +2915,15 @@ sgen_gc_init (void) continue; } + if (!strcmp (opt, "precleaning")) { + precleaning_enabled = TRUE; + continue; + } + if (!strcmp (opt, "no-precleaning")) { + precleaning_enabled = FALSE; + continue; + } + if (major_collector.handle_gc_param && major_collector.handle_gc_param (opt)) continue; @@ -2978,6 +2969,7 @@ sgen_gc_init (void) alloc_nursery (); + sgen_pinning_init (); sgen_cement_init (cement_enabled); if ((env = g_getenv (MONO_GC_DEBUG_NAME))) { @@ -3149,11 +3141,7 @@ sgen_gc_lock (void) void sgen_gc_unlock (void) { - gboolean try_free = sgen_try_free_some_memory; - sgen_try_free_some_memory = FALSE; mono_coop_mutex_unlock (&gc_mutex); - if (try_free) - mono_thread_hazardous_try_free_some (); } void @@ -3204,9 +3192,10 @@ sgen_stop_world (int generation) /* LOCKING: assumes the GC lock is held */ void -sgen_restart_world (int generation, GGTimingInfo *timing) +sgen_restart_world (int generation) { long long major_total = -1, major_marked = -1, los_total = -1, los_marked = -1; + gint64 stw_time; SGEN_ASSERT (0, world_is_stopped, "Why are we restarting a running world?"); @@ -3214,18 +3203,16 @@ sgen_restart_world (int generation, GGTimingInfo *timing) count_cards (&major_total, &major_marked, &los_total, &los_marked); binary_protocol_world_restarting (generation, sgen_timestamp (), major_total, major_marked, los_total, los_marked); - sgen_client_restart_world (generation, timing); - world_is_stopped = FALSE; - binary_protocol_world_restarted (generation, sgen_timestamp ()); + sgen_client_restart_world (generation, &stw_time); - sgen_try_free_some_memory = TRUE; + binary_protocol_world_restarted (generation, sgen_timestamp ()); if (sgen_client_bridge_need_processing ()) sgen_client_bridge_processing_finish (generation); - sgen_memgov_collection_end (generation, timing, timing ? 2 : 0); + sgen_memgov_collection_end (generation, stw_time); } gboolean @@ -3240,7 +3227,7 @@ sgen_check_whole_heap_stw (void) sgen_stop_world (0); sgen_clear_nursery_fragments (); sgen_check_whole_heap (FALSE); - sgen_restart_world (0, NULL); + sgen_restart_world (0); } gint64