-/*
- * sgen-gc.c: Simple generational GC.
+/**
+ * \file
+ * Simple generational GC.
*
* Author:
* Paolo Molaro (lupus@ximian.com)
#include "mono/utils/hazard-pointer.h"
#include <mono/utils/memcheck.h>
+#include <mono/utils/mono-mmap-internals.h>
#undef pthread_create
#undef pthread_join
static guint64 time_minor_pre_collection_fragment_clear = 0;
static guint64 time_minor_pinning = 0;
static guint64 time_minor_scan_remsets = 0;
+static guint64 time_minor_scan_major_blocks = 0;
+static guint64 time_minor_scan_los = 0;
static guint64 time_minor_scan_pinned = 0;
static guint64 time_minor_scan_roots = 0;
static guint64 time_minor_finish_gray_stack = 0;
static void
gray_queue_redirect (SgenGrayQueue *queue)
{
- SGEN_ASSERT (0, concurrent_collection_in_progress, "Where are we redirecting the gray queue to, without a concurrent collection?");
-
sgen_workers_take_from_queue (queue);
}
mono_counters_register ("Minor fragment clear", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_pre_collection_fragment_clear);
mono_counters_register ("Minor pinning", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_pinning);
mono_counters_register ("Minor scan remembered set", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_remsets);
+ mono_counters_register ("Minor scan major blocks", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_major_blocks);
+ mono_counters_register ("Minor scan los", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_los);
mono_counters_register ("Minor scan pinned", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_pinned);
mono_counters_register ("Minor scan roots", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_scan_roots);
mono_counters_register ("Minor fragment creation", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &time_minor_fragment_creation);
return CONTEXT_FROM_OBJECT_OPERATIONS (job->ops, sgen_workers_get_job_gray_queue (worker_data, job->gc_thread_gray_queue));
}
-static void
-job_remembered_set_scan (void *worker_data_untyped, SgenThreadPoolJob *job)
-{
- remset.scan_remsets (scan_copy_context_for_scan_job (worker_data_untyped, (ScanJob*)job));
-}
-
typedef struct {
ScanJob scan_job;
char *heap_start;
scan_finalizer_entries (job_data->queue, ctx);
}
+static void
+job_scan_wbroots (void *worker_data_untyped, SgenThreadPoolJob *job)
+{
+ ScanJob *job_data = (ScanJob*)job;
+ ScanCopyContext ctx = scan_copy_context_for_scan_job (worker_data_untyped, job_data);
+
+ sgen_wbroots_scan_card_table (ctx);
+}
+
+static void
+job_scan_major_card_table (void *worker_data_untyped, SgenThreadPoolJob *job)
+{
+ SGEN_TV_DECLARE (atv);
+ SGEN_TV_DECLARE (btv);
+ ParallelScanJob *job_data = (ParallelScanJob*)job;
+ ScanCopyContext ctx = scan_copy_context_for_scan_job (worker_data_untyped, (ScanJob*)job_data);
+
+ SGEN_TV_GETTIME (atv);
+ major_collector.scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx, job_data->job_index, sgen_workers_get_job_split_count ());
+ SGEN_TV_GETTIME (btv);
+ time_minor_scan_major_blocks += SGEN_TV_ELAPSED (atv, btv);
+}
+
+static void
+job_scan_los_card_table (void *worker_data_untyped, SgenThreadPoolJob *job)
+{
+ SGEN_TV_DECLARE (atv);
+ SGEN_TV_DECLARE (btv);
+ ParallelScanJob *job_data = (ParallelScanJob*)job;
+ ScanCopyContext ctx = scan_copy_context_for_scan_job (worker_data_untyped, (ScanJob*)job_data);
+
+ SGEN_TV_GETTIME (atv);
+ sgen_los_scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx, job_data->job_index, sgen_workers_get_job_split_count ());
+ SGEN_TV_GETTIME (btv);
+ time_minor_scan_los += SGEN_TV_ELAPSED (atv, btv);
+}
+
static void
job_scan_major_mod_union_card_table (void *worker_data_untyped, SgenThreadPoolJob *job)
{
sgen_gray_object_queue_init (gc_thread_gray_queue, NULL, TRUE);
}
+static void
+enqueue_scan_remembered_set_jobs (SgenGrayQueue *gc_thread_gray_queue, SgenObjectOperations *ops, gboolean enqueue)
+{
+ int i, split_count = sgen_workers_get_job_split_count ();
+ ScanJob *sj;
+
+ sj = (ScanJob*)sgen_thread_pool_job_alloc ("scan wbroots", job_scan_wbroots, sizeof (ScanJob));
+ sj->ops = ops;
+ sj->gc_thread_gray_queue = gc_thread_gray_queue;
+ sgen_workers_enqueue_job (&sj->job, enqueue);
+
+ for (i = 0; i < split_count; i++) {
+ ParallelScanJob *psj;
+
+ psj = (ParallelScanJob*)sgen_thread_pool_job_alloc ("scan major remsets", job_scan_major_card_table, sizeof (ParallelScanJob));
+ psj->scan_job.ops = ops;
+ psj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue;
+ psj->job_index = i;
+ sgen_workers_enqueue_job (&psj->scan_job.job, enqueue);
+
+ psj = (ParallelScanJob*)sgen_thread_pool_job_alloc ("scan LOS remsets", job_scan_los_card_table, sizeof (ParallelScanJob));
+ psj->scan_job.ops = ops;
+ psj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue;
+ psj->job_index = i;
+ sgen_workers_enqueue_job (&psj->scan_job.job, enqueue);
+ }
+}
+
static void
enqueue_scan_from_roots_jobs (SgenGrayQueue *gc_thread_gray_queue, char *heap_start, char *heap_end, SgenObjectOperations *ops, gboolean enqueue)
{
static gboolean
collect_nursery (const char *reason, gboolean is_overflow, SgenGrayQueue *unpin_queue)
{
- gboolean needs_major;
+ gboolean needs_major, is_parallel = FALSE;
size_t max_garbage_amount;
char *nursery_next;
mword fragment_total;
- ScanJob *sj;
SgenGrayQueue gc_thread_gray_queue;
- SgenObjectOperations *object_ops;
+ SgenObjectOperations *object_ops_nopar, *object_ops_par = NULL;
ScanCopyContext ctx;
TV_DECLARE (atv);
TV_DECLARE (btv);
binary_protocol_collection_begin (gc_stats.minor_gc_count, GENERATION_NURSERY);
- if (sgen_concurrent_collection_in_progress ())
- object_ops = &sgen_minor_collector.serial_ops_with_concurrent_major;
- else
- object_ops = &sgen_minor_collector.serial_ops;
+ if (sgen_concurrent_collection_in_progress ()) {
+ /* FIXME Support parallel nursery collections with concurrent major */
+ object_ops_nopar = &sgen_minor_collector.serial_ops_with_concurrent_major;
+ } else {
+ object_ops_nopar = &sgen_minor_collector.serial_ops;
+ if (sgen_minor_collector.is_parallel) {
+ object_ops_par = &sgen_minor_collector.parallel_ops;
+ is_parallel = TRUE;
+ }
+ }
if (do_verify_nursery || do_dump_nursery_content)
sgen_debug_verify_nursery (do_dump_nursery_content);
sgen_memgov_minor_collection_start ();
- init_gray_queue (&gc_thread_gray_queue, FALSE);
- ctx = CONTEXT_FROM_OBJECT_OPERATIONS (object_ops, &gc_thread_gray_queue);
+ init_gray_queue (&gc_thread_gray_queue, is_parallel);
+ ctx = CONTEXT_FROM_OBJECT_OPERATIONS (object_ops_nopar, &gc_thread_gray_queue);
gc_stats.minor_gc_count ++;
SGEN_LOG (2, "Finding pinned pointers: %zd in %lld usecs", sgen_get_pinned_count (), (long long)TV_ELAPSED (btv, atv));
SGEN_LOG (4, "Start scan with %zd pinned objects", sgen_get_pinned_count ());
- sj = (ScanJob*)sgen_thread_pool_job_alloc ("scan remset", job_remembered_set_scan, sizeof (ScanJob));
- sj->ops = object_ops;
- sj->gc_thread_gray_queue = &gc_thread_gray_queue;
- sgen_workers_enqueue_job (&sj->job, FALSE);
+ remset.start_scan_remsets ();
+
+ enqueue_scan_remembered_set_jobs (&gc_thread_gray_queue, is_parallel ? object_ops_par : object_ops_nopar, is_parallel);
/* we don't have complete write barrier yet, so we scan all the old generation sections */
TV_GETTIME (btv);
TV_GETTIME (atv);
time_minor_scan_pinned += TV_ELAPSED (btv, atv);
- enqueue_scan_from_roots_jobs (&gc_thread_gray_queue, sgen_get_nursery_start (), nursery_next, object_ops, FALSE);
+ enqueue_scan_from_roots_jobs (&gc_thread_gray_queue, sgen_get_nursery_start (), nursery_next, is_parallel ? object_ops_par : object_ops_nopar, is_parallel);
+
+ if (is_parallel) {
+ gray_queue_redirect (&gc_thread_gray_queue);
+ sgen_workers_start_all_workers (object_ops_nopar, object_ops_par, NULL);
+ sgen_workers_join ();
+ }
TV_GETTIME (btv);
time_minor_scan_roots += TV_ELAPSED (atv, btv);
sgen_gray_object_queue_dispose (&gc_thread_gray_queue);
- remset.finish_minor_collection ();
-
check_scan_starts ();
binary_protocol_flush_buffers (FALSE);
SGEN_ASSERT (0, sgen_workers_all_done (), "Why are the workers not done when we start or finish a major collection?");
if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) {
- sgen_workers_set_num_active_workers (0);
+ if (object_ops_par != NULL)
+ sgen_workers_set_num_active_workers (0);
if (sgen_workers_have_idle_work ()) {
/*
* We force the finish of the worker with the new object ops context
if (mode == COPY_OR_MARK_FROM_ROOTS_FINISH_CONCURRENT) {
int i, split_count = sgen_workers_get_job_split_count ();
+ gboolean parallel = object_ops_par != NULL;
- gray_queue_redirect (gc_thread_gray_queue);
+ /* If we're not parallel we finish the collection on the gc thread */
+ if (parallel)
+ gray_queue_redirect (gc_thread_gray_queue);
/* Mod union card table */
for (i = 0; i < split_count; i++) {
psj = (ParallelScanJob*)sgen_thread_pool_job_alloc ("scan mod union cardtable", job_scan_major_mod_union_card_table, sizeof (ParallelScanJob));
psj->scan_job.ops = object_ops_par ? object_ops_par : object_ops_nopar;
- psj->scan_job.gc_thread_gray_queue = NULL;
+ psj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue;
psj->job_index = i;
- sgen_workers_enqueue_job (&psj->scan_job.job, TRUE);
+ sgen_workers_enqueue_job (&psj->scan_job.job, parallel);
psj = (ParallelScanJob*)sgen_thread_pool_job_alloc ("scan LOS mod union cardtable", job_scan_los_mod_union_card_table, sizeof (ParallelScanJob));
psj->scan_job.ops = object_ops_par ? object_ops_par : object_ops_nopar;
- psj->scan_job.gc_thread_gray_queue = NULL;
+ psj->scan_job.gc_thread_gray_queue = gc_thread_gray_queue;
psj->job_index = i;
- sgen_workers_enqueue_job (&psj->scan_job.job, TRUE);
+ sgen_workers_enqueue_job (&psj->scan_job.job, parallel);
}
- /*
- * If we enqueue a job while workers are running we need to sgen_workers_ensure_awake
- * in order to make sure that we are running the idle func and draining all worker
- * gray queues. The operation of starting workers implies this, so we start them after
- * in order to avoid doing this operation twice. The workers will drain the main gray
- * stack that contained roots and pinned objects and also scan the mod union card
- * table.
- */
- sgen_workers_start_all_workers (object_ops_nopar, object_ops_par, NULL);
- sgen_workers_join ();
+ if (parallel) {
+ /*
+ * If we enqueue a job while workers are running we need to sgen_workers_ensure_awake
+ * in order to make sure that we are running the idle func and draining all worker
+ * gray queues. The operation of starting workers implies this, so we start them after
+ * in order to avoid doing this operation twice. The workers will drain the main gray
+ * stack that contained roots and pinned objects and also scan the mod union card
+ * table.
+ */
+ sgen_workers_start_all_workers (object_ops_nopar, object_ops_par, NULL);
+ sgen_workers_join ();
+ }
}
sgen_pin_stats_report ();
* the conservative scan, otherwise by the remembered set scan.
*/
+/**
+ * mono_gc_wbarrier_arrayref_copy:
+ */
void
mono_gc_wbarrier_arrayref_copy (gpointer dest_ptr, gpointer src_ptr, int count)
{
remset.wbarrier_arrayref_copy (dest_ptr, src_ptr, count);
}
+/**
+ * mono_gc_wbarrier_generic_nostore:
+ */
void
mono_gc_wbarrier_generic_nostore (gpointer ptr)
{
remset.wbarrier_generic_nostore (ptr);
}
+/**
+ * mono_gc_wbarrier_generic_store:
+ */
void
mono_gc_wbarrier_generic_store (gpointer ptr, GCObject* value)
{
sgen_dummy_use (value);
}
-/* Same as mono_gc_wbarrier_generic_store () but performs the store
+/**
+ * mono_gc_wbarrier_generic_store_atomic:
+ * Same as \c mono_gc_wbarrier_generic_store but performs the store
* as an atomic operation with release semantics.
*/
void
void
sgen_gc_init (void)
{
- const char *env;
+ char *env;
char **opts, **ptr;
char *major_collector_opt = NULL;
char *minor_collector_opt = NULL;
if ((env = g_getenv (MONO_GC_PARAMS_NAME)) || gc_params_options) {
params_opts = g_strdup_printf ("%s,%s", gc_params_options ? gc_params_options : "", env ? env : "");
+ g_free (env);
}
if (params_opts) {
sgen_client_init ();
if (!minor_collector_opt) {
- sgen_simple_nursery_init (&sgen_minor_collector);
+ sgen_simple_nursery_init (&sgen_minor_collector, FALSE);
} else {
if (!strcmp (minor_collector_opt, "simple")) {
use_simple_nursery:
- sgen_simple_nursery_init (&sgen_minor_collector);
+ sgen_simple_nursery_init (&sgen_minor_collector, FALSE);
+ } else if (!strcmp (minor_collector_opt, "simple-par")) {
+ sgen_simple_nursery_init (&sgen_minor_collector, TRUE);
} else if (!strcmp (minor_collector_opt, "split")) {
sgen_split_nursery_init (&sgen_minor_collector);
} else {
if ((env = g_getenv (MONO_GC_DEBUG_NAME)) || gc_debug_options) {
debug_opts = g_strdup_printf ("%s,%s", gc_debug_options ? gc_debug_options : "", env ? env : "");
+ g_free (env);
}
if (debug_opts) {
} else if (!strcmp (opt, "verify-before-allocs")) {
verify_before_allocs = 1;
has_per_allocation_action = TRUE;
+ } else if (g_str_has_prefix (opt, "max-valloc-size=")) {
+ size_t max_valloc_size;
+ char *arg = strchr (opt, '=') + 1;
+ if (*opt && mono_gc_parse_environment_string_extract_number (arg, &max_valloc_size)) {
+ mono_valloc_set_limit (max_valloc_size);
+ } else {
+ sgen_env_var_error (MONO_GC_DEBUG_NAME, NULL, "`max-valloc-size` must be an integer.");
+ }
+ continue;
} else if (g_str_has_prefix (opt, "verify-before-allocs=")) {
char *arg = strchr (opt, '=') + 1;
verify_before_allocs = atoi (arg);
fprintf (stderr, "Valid <option>s are:\n");
fprintf (stderr, " collect-before-allocs[=<n>]\n");
fprintf (stderr, " verify-before-allocs[=<n>]\n");
+ fprintf (stderr, " max-valloc-size=N (where N is an integer, possibly with a k, m or a g suffix)\n");
fprintf (stderr, " check-remset-consistency\n");
fprintf (stderr, " check-mark-bits\n");
fprintf (stderr, " check-nursery-pinned\n");
if (major_collector.post_param_init)
major_collector.post_param_init (&major_collector);
- if (major_collector.needs_thread_pool) {
+ if (major_collector.is_concurrent || sgen_minor_collector.is_parallel) {
int num_workers = 1;
- if (major_collector.is_parallel) {
+ if (major_collector.is_parallel || sgen_minor_collector.is_parallel) {
/* FIXME Detect the number of physical cores, instead of logical */
num_workers = mono_cpu_count () / 2;
if (num_workers < 1)
return &major_collector;
}
+SgenMinorCollector*
+sgen_get_minor_collector (void)
+{
+ return &sgen_minor_collector;
+}
+
SgenRememberedSet*
sgen_get_remset (void)
{
projects / mono.git / blobdiff